2024-12-11 20:34:03,679 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-11 20:34:03,689 main DEBUG Took 0.008762 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-11 20:34:03,690 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-11 20:34:03,690 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-11 20:34:03,691 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-11 20:34:03,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,698 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-11 20:34:03,709 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,710 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,711 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,712 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,712 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,714 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,714 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,715 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,715 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,716 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,716 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,717 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,717 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,718 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,719 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,719 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,720 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,720 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,721 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,721 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,722 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,722 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-11 20:34:03,723 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,723 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-11 20:34:03,725 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-11 20:34:03,726 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-11 20:34:03,729 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-11 20:34:03,730 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-11 20:34:03,731 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-11 20:34:03,732 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-11 20:34:03,741 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-11 20:34:03,744 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-11 20:34:03,745 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-11 20:34:03,746 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-11 20:34:03,746 main DEBUG createAppenders(={Console}) 2024-12-11 20:34:03,747 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-11 20:34:03,747 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-11 20:34:03,747 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-11 20:34:03,748 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-11 20:34:03,748 main DEBUG OutputStream closed 2024-12-11 20:34:03,749 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-11 20:34:03,749 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-11 20:34:03,750 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-11 20:34:03,834 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-11 20:34:03,837 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-11 20:34:03,839 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-11 20:34:03,840 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-11 20:34:03,841 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-11 20:34:03,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-11 20:34:03,842 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-11 20:34:03,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-11 20:34:03,843 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-11 20:34:03,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-11 20:34:03,844 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-11 20:34:03,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-11 20:34:03,845 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-11 20:34:03,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-11 20:34:03,846 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-11 20:34:03,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-11 20:34:03,847 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-11 20:34:03,848 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-11 20:34:03,851 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11 20:34:03,852 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-11 20:34:03,852 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-11 20:34:03,853 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-11T20:34:04,122 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9 2024-12-11 20:34:04,125 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-11 20:34:04,126 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-11T20:34:04,134 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-11T20:34:04,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-11T20:34:04,171 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/cluster_cff584a6-4744-f919-71a4-2a9a95c09adc, deleteOnExit=true 2024-12-11T20:34:04,171 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-11T20:34:04,172 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/test.cache.data in system properties and HBase conf 2024-12-11T20:34:04,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/hadoop.tmp.dir in system properties and HBase conf 2024-12-11T20:34:04,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/hadoop.log.dir in system properties and HBase conf 2024-12-11T20:34:04,174 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-11T20:34:04,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-11T20:34:04,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-11T20:34:04,269 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-11T20:34:04,356 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-11T20:34:04,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-11T20:34:04,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-11T20:34:04,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-11T20:34:04,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T20:34:04,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-11T20:34:04,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-11T20:34:04,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-11T20:34:04,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T20:34:04,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-11T20:34:04,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/nfs.dump.dir in system properties and HBase conf 2024-12-11T20:34:04,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/java.io.tmpdir in system properties and HBase conf 2024-12-11T20:34:04,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-11T20:34:04,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-11T20:34:04,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-11T20:34:05,271 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-11T20:34:05,339 INFO [Time-limited test {}] log.Log(170): Logging initialized @2309ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-11T20:34:05,407 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T20:34:05,461 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T20:34:05,479 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T20:34:05,479 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T20:34:05,480 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-11T20:34:05,491 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T20:34:05,493 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4516de21{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/hadoop.log.dir/,AVAILABLE} 2024-12-11T20:34:05,494 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6efc0ba4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T20:34:05,659 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@55a6f9c7{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/java.io.tmpdir/jetty-localhost-35227-hadoop-hdfs-3_4_1-tests_jar-_-any-18398364489104236659/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T20:34:05,665 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@40742525{HTTP/1.1, (http/1.1)}{localhost:35227} 2024-12-11T20:34:05,666 INFO [Time-limited test {}] server.Server(415): Started @2636ms 2024-12-11T20:34:06,169 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-11T20:34:06,175 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-11T20:34:06,176 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-11T20:34:06,176 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-11T20:34:06,176 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-11T20:34:06,177 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@15953f85{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/hadoop.log.dir/,AVAILABLE} 2024-12-11T20:34:06,177 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@55a78c31{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-11T20:34:06,272 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3874740c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/java.io.tmpdir/jetty-localhost-39479-hadoop-hdfs-3_4_1-tests_jar-_-any-15195887298664494603/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T20:34:06,272 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@589acbe1{HTTP/1.1, (http/1.1)}{localhost:39479} 2024-12-11T20:34:06,273 INFO [Time-limited test {}] server.Server(415): Started @3243ms 2024-12-11T20:34:06,319 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-11T20:34:07,100 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/cluster_cff584a6-4744-f919-71a4-2a9a95c09adc/data/data1/current/BP-2058600437-172.17.0.2-1733949244852/current, will proceed with Du for space computation calculation, 2024-12-11T20:34:07,100 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/cluster_cff584a6-4744-f919-71a4-2a9a95c09adc/data/data2/current/BP-2058600437-172.17.0.2-1733949244852/current, will proceed with Du for space computation calculation, 2024-12-11T20:34:07,125 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-11T20:34:07,165 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43a3365c0d3e2cd4 with lease ID 0xc07e5457252536dc: Processing first storage report for DS-886c8bf1-d90e-48ce-99f1-283c01a189d6 from datanode DatanodeRegistration(127.0.0.1:35965, datanodeUuid=a72c7df4-e10e-48ba-9eff-506085311529, infoPort=45809, infoSecurePort=0, ipcPort=45339, storageInfo=lv=-57;cid=testClusterID;nsid=456175055;c=1733949244852) 2024-12-11T20:34:07,166 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43a3365c0d3e2cd4 with lease ID 0xc07e5457252536dc: from storage DS-886c8bf1-d90e-48ce-99f1-283c01a189d6 node DatanodeRegistration(127.0.0.1:35965, datanodeUuid=a72c7df4-e10e-48ba-9eff-506085311529, infoPort=45809, infoSecurePort=0, ipcPort=45339, storageInfo=lv=-57;cid=testClusterID;nsid=456175055;c=1733949244852), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-11T20:34:07,166 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x43a3365c0d3e2cd4 with lease ID 0xc07e5457252536dc: Processing first storage report for DS-e7a7adda-96a9-41d2-8358-c6c93cfd2370 from datanode DatanodeRegistration(127.0.0.1:35965, datanodeUuid=a72c7df4-e10e-48ba-9eff-506085311529, infoPort=45809, infoSecurePort=0, ipcPort=45339, storageInfo=lv=-57;cid=testClusterID;nsid=456175055;c=1733949244852) 2024-12-11T20:34:07,167 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x43a3365c0d3e2cd4 with lease ID 0xc07e5457252536dc: from storage DS-e7a7adda-96a9-41d2-8358-c6c93cfd2370 node DatanodeRegistration(127.0.0.1:35965, datanodeUuid=a72c7df4-e10e-48ba-9eff-506085311529, infoPort=45809, infoSecurePort=0, ipcPort=45339, storageInfo=lv=-57;cid=testClusterID;nsid=456175055;c=1733949244852), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-11T20:34:07,175 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9 2024-12-11T20:34:07,237 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/cluster_cff584a6-4744-f919-71a4-2a9a95c09adc/zookeeper_0, clientPort=59301, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/cluster_cff584a6-4744-f919-71a4-2a9a95c09adc/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/cluster_cff584a6-4744-f919-71a4-2a9a95c09adc/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-11T20:34:07,247 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59301 2024-12-11T20:34:07,255 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:34:07,258 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:34:07,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741825_1001 (size=7) 2024-12-11T20:34:07,867 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043 with version=8 2024-12-11T20:34:07,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/hbase-staging 2024-12-11T20:34:07,945 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-11T20:34:08,189 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5be53b084ac7:0 server-side Connection retries=45 2024-12-11T20:34:08,198 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T20:34:08,198 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T20:34:08,202 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T20:34:08,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T20:34:08,202 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T20:34:08,314 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-11T20:34:08,366 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-11T20:34:08,376 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-11T20:34:08,380 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T20:34:08,407 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 85866 (auto-detected) 2024-12-11T20:34:08,408 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-11T20:34:08,425 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39923 2024-12-11T20:34:08,442 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39923 connecting to ZooKeeper ensemble=127.0.0.1:59301 2024-12-11T20:34:08,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:399230x0, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T20:34:08,550 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39923-0x10016a38d920000 connected 2024-12-11T20:34:08,644 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:34:08,647 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:34:08,656 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T20:34:08,660 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043, hbase.cluster.distributed=false 2024-12-11T20:34:08,680 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T20:34:08,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39923 2024-12-11T20:34:08,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39923 2024-12-11T20:34:08,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39923 2024-12-11T20:34:08,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39923 2024-12-11T20:34:08,686 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39923 2024-12-11T20:34:08,784 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5be53b084ac7:0 server-side Connection retries=45 2024-12-11T20:34:08,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T20:34:08,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-11T20:34:08,786 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-11T20:34:08,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-11T20:34:08,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-11T20:34:08,789 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-11T20:34:08,791 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-11T20:34:08,792 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44543 2024-12-11T20:34:08,794 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44543 connecting to ZooKeeper ensemble=127.0.0.1:59301 2024-12-11T20:34:08,795 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:34:08,797 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:34:08,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:445430x0, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-11T20:34:08,826 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:445430x0, quorum=127.0.0.1:59301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T20:34:08,827 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44543-0x10016a38d920001 connected 2024-12-11T20:34:08,832 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-11T20:34:08,840 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-11T20:34:08,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-11T20:34:08,849 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-11T20:34:08,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44543 2024-12-11T20:34:08,850 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44543 2024-12-11T20:34:08,851 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44543 2024-12-11T20:34:08,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44543 2024-12-11T20:34:08,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44543 2024-12-11T20:34:08,874 DEBUG [M:0;5be53b084ac7:39923 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5be53b084ac7:39923 2024-12-11T20:34:08,875 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5be53b084ac7,39923,1733949248041 2024-12-11T20:34:08,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T20:34:08,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T20:34:08,903 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5be53b084ac7,39923,1733949248041 2024-12-11T20:34:08,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-11T20:34:08,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:08,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:08,934 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-11T20:34:08,935 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5be53b084ac7,39923,1733949248041 from backup master directory 2024-12-11T20:34:08,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5be53b084ac7,39923,1733949248041 2024-12-11T20:34:08,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T20:34:08,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-11T20:34:08,943 WARN [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T20:34:08,943 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5be53b084ac7,39923,1733949248041 2024-12-11T20:34:08,945 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-11T20:34:08,947 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-11T20:34:08,999 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/hbase.id] with ID: 2e38c395-40d3-42c4-b93e-358d4c1c19bd 2024-12-11T20:34:08,999 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/.tmp/hbase.id 2024-12-11T20:34:09,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741826_1002 (size=42) 2024-12-11T20:34:09,412 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/.tmp/hbase.id]:[hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/hbase.id] 2024-12-11T20:34:09,463 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-11T20:34:09,470 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-11T20:34:09,486 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-11T20:34:09,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:09,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:09,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741827_1003 (size=196) 2024-12-11T20:34:09,980 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:34:09,982 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-11T20:34:09,996 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:10,000 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T20:34:10,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741828_1004 (size=1189) 2024-12-11T20:34:10,452 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store 2024-12-11T20:34:10,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741829_1005 (size=34) 2024-12-11T20:34:10,879 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-11T20:34:10,883 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:34:10,884 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T20:34:10,884 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:34:10,885 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:34:10,886 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T20:34:10,886 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:34:10,886 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:34:10,888 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733949250884Disabling compacts and flushes for region at 1733949250884Disabling writes for close at 1733949250886 (+2 ms)Writing region close event to WAL at 1733949250886Closed at 1733949250886 2024-12-11T20:34:10,890 WARN [master/5be53b084ac7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/.initializing 2024-12-11T20:34:10,890 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/WALs/5be53b084ac7,39923,1733949248041 2024-12-11T20:34:10,898 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T20:34:10,913 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5be53b084ac7%2C39923%2C1733949248041, suffix=, logDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/WALs/5be53b084ac7,39923,1733949248041, archiveDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/oldWALs, maxLogs=10 2024-12-11T20:34:10,933 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/WALs/5be53b084ac7,39923,1733949248041/5be53b084ac7%2C39923%2C1733949248041.1733949250917, exclude list is [], retry=0 2024-12-11T20:34:10,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35965,DS-886c8bf1-d90e-48ce-99f1-283c01a189d6,DISK] 2024-12-11T20:34:10,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-11T20:34:10,983 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/WALs/5be53b084ac7,39923,1733949248041/5be53b084ac7%2C39923%2C1733949248041.1733949250917 2024-12-11T20:34:10,984 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45809:45809)] 2024-12-11T20:34:10,985 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:34:10,985 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:34:10,988 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:10,989 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:11,022 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:11,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-11T20:34:11,046 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:11,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:34:11,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:11,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-11T20:34:11,053 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:11,054 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:11,054 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:11,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-11T20:34:11,057 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:11,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:11,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:11,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-11T20:34:11,060 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:11,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:11,061 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:11,065 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:11,066 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:11,072 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:11,073 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:11,077 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-11T20:34:11,082 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-11T20:34:11,087 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:34:11,088 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68497786, jitterRate=0.020696550607681274}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-11T20:34:11,093 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733949250999Initializing all the Stores at 1733949251001 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733949251001Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949251002 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949251002Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949251002Cleaning up temporary data from old regions at 1733949251073 (+71 ms)Region opened successfully at 1733949251093 (+20 ms) 2024-12-11T20:34:11,094 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-11T20:34:11,127 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2388c164, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5be53b084ac7/172.17.0.2:0 2024-12-11T20:34:11,153 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-11T20:34:11,163 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-11T20:34:11,163 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-11T20:34:11,165 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-11T20:34:11,166 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-11T20:34:11,171 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-11T20:34:11,171 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-11T20:34:11,193 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-11T20:34:11,201 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-11T20:34:11,259 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-11T20:34:11,263 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-11T20:34:11,266 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-11T20:34:11,275 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-11T20:34:11,277 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-11T20:34:11,281 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-11T20:34:11,291 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-11T20:34:11,293 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-11T20:34:11,300 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-11T20:34:11,322 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-11T20:34:11,333 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-11T20:34:11,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T20:34:11,341 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-11T20:34:11,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:11,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:11,345 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5be53b084ac7,39923,1733949248041, sessionid=0x10016a38d920000, setting cluster-up flag (Was=false) 2024-12-11T20:34:11,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:11,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:11,400 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-11T20:34:11,404 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5be53b084ac7,39923,1733949248041 2024-12-11T20:34:11,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:11,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:11,450 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-11T20:34:11,454 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5be53b084ac7,39923,1733949248041 2024-12-11T20:34:11,461 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-11T20:34:11,530 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-11T20:34:11,538 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-11T20:34:11,545 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-11T20:34:11,550 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5be53b084ac7,39923,1733949248041 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-11T20:34:11,558 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5be53b084ac7:0, corePoolSize=5, maxPoolSize=5 2024-12-11T20:34:11,558 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5be53b084ac7:0, corePoolSize=5, maxPoolSize=5 2024-12-11T20:34:11,558 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5be53b084ac7:0, corePoolSize=5, maxPoolSize=5 2024-12-11T20:34:11,559 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5be53b084ac7:0, corePoolSize=5, maxPoolSize=5 2024-12-11T20:34:11,559 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5be53b084ac7:0, corePoolSize=10, maxPoolSize=10 2024-12-11T20:34:11,559 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,559 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5be53b084ac7:0, corePoolSize=2, maxPoolSize=2 2024-12-11T20:34:11,559 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,561 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733949281561 2024-12-11T20:34:11,562 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-11T20:34:11,563 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(746): ClusterId : 2e38c395-40d3-42c4-b93e-358d4c1c19bd 2024-12-11T20:34:11,563 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-11T20:34:11,565 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T20:34:11,566 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-11T20:34:11,566 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-11T20:34:11,567 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-11T20:34:11,567 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-11T20:34:11,567 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-11T20:34:11,568 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-11T20:34:11,568 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,571 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-11T20:34:11,572 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-11T20:34:11,573 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-11T20:34:11,573 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:11,573 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-11T20:34:11,574 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-11T20:34:11,575 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-11T20:34:11,577 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5be53b084ac7:0:becomeActiveMaster-HFileCleaner.large.0-1733949251576,5,FailOnTimeoutGroup] 2024-12-11T20:34:11,578 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5be53b084ac7:0:becomeActiveMaster-HFileCleaner.small.0-1733949251577,5,FailOnTimeoutGroup] 2024-12-11T20:34:11,578 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,578 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-11T20:34:11,579 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,579 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741831_1007 (size=1321) 2024-12-11T20:34:11,602 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-11T20:34:11,603 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-11T20:34:11,610 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-11T20:34:11,610 DEBUG [RS:0;5be53b084ac7:44543 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cbccb4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5be53b084ac7/172.17.0.2:0 2024-12-11T20:34:11,623 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5be53b084ac7:44543 2024-12-11T20:34:11,626 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-11T20:34:11,626 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-11T20:34:11,626 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-11T20:34:11,629 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(2659): reportForDuty to master=5be53b084ac7,39923,1733949248041 with port=44543, startcode=1733949248752 2024-12-11T20:34:11,639 DEBUG [RS:0;5be53b084ac7:44543 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-11T20:34:11,665 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54513, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-11T20:34:11,670 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39923 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5be53b084ac7,44543,1733949248752 2024-12-11T20:34:11,672 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39923 {}] master.ServerManager(517): Registering regionserver=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:11,684 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043 2024-12-11T20:34:11,684 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34529 2024-12-11T20:34:11,684 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-11T20:34:11,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T20:34:11,716 DEBUG [RS:0;5be53b084ac7:44543 {}] zookeeper.ZKUtil(111): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5be53b084ac7,44543,1733949248752 2024-12-11T20:34:11,717 WARN [RS:0;5be53b084ac7:44543 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-11T20:34:11,717 INFO [RS:0;5be53b084ac7:44543 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T20:34:11,717 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/WALs/5be53b084ac7,44543,1733949248752 2024-12-11T20:34:11,719 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5be53b084ac7,44543,1733949248752] 2024-12-11T20:34:11,739 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-11T20:34:11,751 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-11T20:34:11,756 INFO [RS:0;5be53b084ac7:44543 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-11T20:34:11,756 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,757 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-11T20:34:11,762 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-11T20:34:11,763 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,763 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,764 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,764 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,764 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,764 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,764 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5be53b084ac7:0, corePoolSize=2, maxPoolSize=2 2024-12-11T20:34:11,764 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,764 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,765 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,765 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,765 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,765 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5be53b084ac7:0, corePoolSize=1, maxPoolSize=1 2024-12-11T20:34:11,765 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5be53b084ac7:0, corePoolSize=3, maxPoolSize=3 2024-12-11T20:34:11,765 DEBUG [RS:0;5be53b084ac7:44543 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0, corePoolSize=3, maxPoolSize=3 2024-12-11T20:34:11,770 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,770 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,770 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,770 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,770 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,770 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(168): Chore ScheduledChore name=5be53b084ac7,44543,1733949248752-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T20:34:11,787 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.HeapMemoryManager(213): Starting, tuneOn=false 2024-12-11T20:34:11,789 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(168): Chore ScheduledChore name=5be53b084ac7,44543,1733949248752-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,789 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,789 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.Replication(171): 5be53b084ac7,44543,1733949248752 started 2024-12-11T20:34:11,804 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:11,805 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1482): Serving as 5be53b084ac7,44543,1733949248752, RpcServer on 5be53b084ac7/172.17.0.2:44543, sessionid=0x10016a38d920001 2024-12-11T20:34:11,806 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-11T20:34:11,806 DEBUG [RS:0;5be53b084ac7:44543 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5be53b084ac7,44543,1733949248752 2024-12-11T20:34:11,806 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5be53b084ac7,44543,1733949248752' 2024-12-11T20:34:11,806 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-11T20:34:11,807 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-11T20:34:11,808 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-11T20:34:11,808 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-11T20:34:11,808 DEBUG [RS:0;5be53b084ac7:44543 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5be53b084ac7,44543,1733949248752 2024-12-11T20:34:11,808 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5be53b084ac7,44543,1733949248752' 2024-12-11T20:34:11,808 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-11T20:34:11,809 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-11T20:34:11,809 DEBUG [RS:0;5be53b084ac7:44543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-11T20:34:11,809 INFO [RS:0;5be53b084ac7:44543 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-11T20:34:11,809 INFO [RS:0;5be53b084ac7:44543 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-11T20:34:11,923 INFO [RS:0;5be53b084ac7:44543 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-11T20:34:11,926 INFO [RS:0;5be53b084ac7:44543 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5be53b084ac7%2C44543%2C1733949248752, suffix=, logDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/WALs/5be53b084ac7,44543,1733949248752, archiveDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/oldWALs, maxLogs=32 2024-12-11T20:34:11,939 DEBUG [RS:0;5be53b084ac7:44543 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/WALs/5be53b084ac7,44543,1733949248752/5be53b084ac7%2C44543%2C1733949248752.1733949251928, exclude list is [], retry=0 2024-12-11T20:34:11,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35965,DS-886c8bf1-d90e-48ce-99f1-283c01a189d6,DISK] 2024-12-11T20:34:11,947 INFO [RS:0;5be53b084ac7:44543 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/WALs/5be53b084ac7,44543,1733949248752/5be53b084ac7%2C44543%2C1733949248752.1733949251928 2024-12-11T20:34:11,947 DEBUG [RS:0;5be53b084ac7:44543 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45809:45809)] 2024-12-11T20:34:11,988 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-11T20:34:11,989 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043 2024-12-11T20:34:12,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741833_1009 (size=32) 2024-12-11T20:34:12,408 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:34:12,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T20:34:12,417 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T20:34:12,417 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:12,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:34:12,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T20:34:12,422 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T20:34:12,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:12,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:34:12,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T20:34:12,426 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T20:34:12,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:12,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:34:12,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T20:34:12,430 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T20:34:12,430 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:12,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:34:12,431 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T20:34:12,432 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740 2024-12-11T20:34:12,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740 2024-12-11T20:34:12,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T20:34:12,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T20:34:12,437 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T20:34:12,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T20:34:12,444 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:34:12,445 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71309131, jitterRate=0.06258885562419891}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T20:34:12,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733949252408Initializing all the Stores at 1733949252412 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733949252412Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733949252413 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949252413Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733949252413Cleaning up temporary data from old regions at 1733949252436 (+23 ms)Region opened successfully at 1733949252448 (+12 ms) 2024-12-11T20:34:12,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T20:34:12,448 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T20:34:12,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T20:34:12,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T20:34:12,448 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T20:34:12,450 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T20:34:12,450 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733949252448Disabling compacts and flushes for region at 1733949252448Disabling writes for close at 1733949252448Writing region close event to WAL at 1733949252449 (+1 ms)Closed at 1733949252450 (+1 ms) 2024-12-11T20:34:12,454 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T20:34:12,454 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-11T20:34:12,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-11T20:34:12,469 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T20:34:12,472 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-11T20:34:12,626 DEBUG [5be53b084ac7:39923 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-11T20:34:12,640 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:12,645 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5be53b084ac7,44543,1733949248752, state=OPENING 2024-12-11T20:34:12,700 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-11T20:34:12,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:12,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:34:12,709 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T20:34:12,709 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T20:34:12,710 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-11T20:34:12,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:34:12,889 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-11T20:34:12,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53797, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-11T20:34:12,903 INFO [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-11T20:34:12,903 INFO [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-11T20:34:12,904 INFO [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-11T20:34:12,907 INFO [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5be53b084ac7%2C44543%2C1733949248752.meta, suffix=.meta, logDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/WALs/5be53b084ac7,44543,1733949248752, archiveDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/oldWALs, maxLogs=32 2024-12-11T20:34:12,921 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/WALs/5be53b084ac7,44543,1733949248752/5be53b084ac7%2C44543%2C1733949248752.meta.1733949252908.meta, exclude list is [], retry=0 2024-12-11T20:34:12,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35965,DS-886c8bf1-d90e-48ce-99f1-283c01a189d6,DISK] 2024-12-11T20:34:12,927 INFO [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/WALs/5be53b084ac7,44543,1733949248752/5be53b084ac7%2C44543%2C1733949248752.meta.1733949252908.meta 2024-12-11T20:34:12,928 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45809:45809)] 2024-12-11T20:34:12,928 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:34:12,930 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-11T20:34:12,932 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-11T20:34:12,936 INFO [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-11T20:34:12,940 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-11T20:34:12,941 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:34:12,941 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-11T20:34:12,941 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-11T20:34:12,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-11T20:34:12,946 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-11T20:34:12,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:12,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:34:12,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-11T20:34:12,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-11T20:34:12,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:12,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:34:12,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-11T20:34:12,951 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-11T20:34:12,951 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:12,951 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:34:12,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-11T20:34:12,953 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-11T20:34:12,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:12,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-11T20:34:12,954 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-11T20:34:12,955 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740 2024-12-11T20:34:12,958 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740 2024-12-11T20:34:12,960 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-11T20:34:12,961 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-11T20:34:12,962 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T20:34:12,964 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-11T20:34:12,966 INFO [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61473016, jitterRate=-0.08398067951202393}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T20:34:12,966 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-11T20:34:12,967 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733949252942Writing region info on filesystem at 1733949252942Initializing all the Stores at 1733949252944 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733949252944Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733949252944Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949252944Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733949252944Cleaning up temporary data from old regions at 1733949252961 (+17 ms)Running coprocessor post-open hooks at 1733949252966 (+5 ms)Region opened successfully at 1733949252967 (+1 ms) 2024-12-11T20:34:12,974 INFO [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733949252880 2024-12-11T20:34:12,985 DEBUG [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-11T20:34:12,985 INFO [RS_OPEN_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-11T20:34:12,987 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:12,989 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5be53b084ac7,44543,1733949248752, state=OPEN 2024-12-11T20:34:13,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T20:34:13,036 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-11T20:34:13,036 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T20:34:13,036 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-11T20:34:13,037 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:13,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-11T20:34:13,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5be53b084ac7,44543,1733949248752 in 325 msec 2024-12-11T20:34:13,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-11T20:34:13,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 583 msec 2024-12-11T20:34:13,053 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-11T20:34:13,053 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-11T20:34:13,071 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:13,072 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:13,092 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:13,095 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52247, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:13,115 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6240 sec 2024-12-11T20:34:13,115 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733949253115, completionTime=-1 2024-12-11T20:34:13,118 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-11T20:34:13,118 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-11T20:34:13,142 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-11T20:34:13,142 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733949313142 2024-12-11T20:34:13,142 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733949373142 2024-12-11T20:34:13,142 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-11T20:34:13,144 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5be53b084ac7,39923,1733949248041-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:13,145 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5be53b084ac7,39923,1733949248041-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:13,145 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5be53b084ac7,39923,1733949248041-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:13,146 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5be53b084ac7:39923, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:13,146 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:13,147 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:13,152 DEBUG [master/5be53b084ac7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-11T20:34:13,173 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.230sec 2024-12-11T20:34:13,174 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-11T20:34:13,175 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-11T20:34:13,176 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-11T20:34:13,177 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-11T20:34:13,177 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-11T20:34:13,178 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5be53b084ac7,39923,1733949248041-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-11T20:34:13,178 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5be53b084ac7,39923,1733949248041-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-11T20:34:13,187 DEBUG [master/5be53b084ac7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-11T20:34:13,188 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-11T20:34:13,188 INFO [master/5be53b084ac7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5be53b084ac7,39923,1733949248041-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-11T20:34:13,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25af98c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:13,280 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-11T20:34:13,280 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-11T20:34:13,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:13,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:13,291 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15e8df74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:13,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:13,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:13,300 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:13,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@668ecb5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:13,303 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:13,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:13,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:13,313 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:13,347 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5be53b084ac7,39923,1733949248041 2024-12-11T20:34:13,361 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=183, ProcessCount=11, AvailableMemoryMB=2876 2024-12-11T20:34:13,370 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T20:34:13,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5be53b084ac7,39923,1733949248041 2024-12-11T20:34:13,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@216dfbe8 2024-12-11T20:34:13,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T20:34:13,384 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T20:34:13,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T20:34:13,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:34:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T20:34:13,403 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:34:13,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-11T20:34:13,406 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:13,408 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:34:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T20:34:13,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741835_1011 (size=1061) 2024-12-11T20:34:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T20:34:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T20:34:13,832 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043 2024-12-11T20:34:13,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741836_1012 (size=53) 2024-12-11T20:34:14,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T20:34:14,247 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:34:14,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c0a19b15b87adee237235cecb1e4c99f, disabling compactions & flushes 2024-12-11T20:34:14,248 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:14,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:14,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. after waiting 0 ms 2024-12-11T20:34:14,248 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:14,249 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:14,249 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0a19b15b87adee237235cecb1e4c99f: Waiting for close lock at 1733949254248Disabling compacts and flushes for region at 1733949254248Disabling writes for close at 1733949254248Writing region close event to WAL at 1733949254249 (+1 ms)Closed at 1733949254249 2024-12-11T20:34:14,252 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:34:14,259 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733949254252"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733949254252"}]},"ts":"1733949254252"} 2024-12-11T20:34:14,263 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:34:14,265 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:34:14,268 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949254265"}]},"ts":"1733949254265"} 2024-12-11T20:34:14,271 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T20:34:14,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0a19b15b87adee237235cecb1e4c99f, ASSIGN}] 2024-12-11T20:34:14,275 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0a19b15b87adee237235cecb1e4c99f, ASSIGN 2024-12-11T20:34:14,277 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0a19b15b87adee237235cecb1e4c99f, ASSIGN; state=OFFLINE, location=5be53b084ac7,44543,1733949248752; forceNewPlan=false, retain=false 2024-12-11T20:34:14,429 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0a19b15b87adee237235cecb1e4c99f, regionState=OPENING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:14,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0a19b15b87adee237235cecb1e4c99f, ASSIGN because future has completed 2024-12-11T20:34:14,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:34:14,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T20:34:14,606 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:14,606 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:34:14,607 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,607 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:34:14,607 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,607 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,610 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,613 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:34:14,614 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0a19b15b87adee237235cecb1e4c99f columnFamilyName A 2024-12-11T20:34:14,614 DEBUG [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:14,615 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] regionserver.HStore(327): Store=c0a19b15b87adee237235cecb1e4c99f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:14,615 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,618 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:34:14,618 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0a19b15b87adee237235cecb1e4c99f columnFamilyName B 2024-12-11T20:34:14,618 DEBUG [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:14,619 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] regionserver.HStore(327): Store=c0a19b15b87adee237235cecb1e4c99f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:14,620 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,622 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:34:14,623 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0a19b15b87adee237235cecb1e4c99f columnFamilyName C 2024-12-11T20:34:14,623 DEBUG [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:14,624 INFO [StoreOpener-c0a19b15b87adee237235cecb1e4c99f-1 {}] regionserver.HStore(327): Store=c0a19b15b87adee237235cecb1e4c99f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:14,624 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:14,624 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,626 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,627 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,629 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,630 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,631 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T20:34:14,633 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,637 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:34:14,638 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c0a19b15b87adee237235cecb1e4c99f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61900996, jitterRate=-0.077603280544281}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T20:34:14,638 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:14,638 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c0a19b15b87adee237235cecb1e4c99f: Running coprocessor pre-open hook at 1733949254607Writing region info on filesystem at 1733949254607Initializing all the Stores at 1733949254609 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949254609Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949254609Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949254610 (+1 ms)Cleaning up temporary data from old regions at 1733949254630 (+20 ms)Running coprocessor post-open hooks at 1733949254638 (+8 ms)Region opened successfully at 1733949254638 2024-12-11T20:34:14,641 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., pid=6, masterSystemTime=1733949254593 2024-12-11T20:34:14,644 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:14,644 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:14,645 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c0a19b15b87adee237235cecb1e4c99f, regionState=OPEN, openSeqNum=2, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:14,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:34:14,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-11T20:34:14,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 in 213 msec 2024-12-11T20:34:14,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-11T20:34:14,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0a19b15b87adee237235cecb1e4c99f, ASSIGN in 382 msec 2024-12-11T20:34:14,662 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:34:14,662 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949254662"}]},"ts":"1733949254662"} 2024-12-11T20:34:14,665 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T20:34:14,667 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:34:14,671 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2730 sec 2024-12-11T20:34:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-11T20:34:15,560 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:15,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f9df9bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:15,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:15,569 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6a9f59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:15,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:15,573 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@708c62f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,576 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:15,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:15,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:15,584 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T20:34:15,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 5be53b084ac7,39923,1733949248041 2024-12-11T20:34:15,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4e704312 2024-12-11T20:34:15,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T20:34:15,589 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T20:34:15,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@161d9329, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:15,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:15,596 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:15,596 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a183203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:15,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:15,599 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:15,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef8cf77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3132aa0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:15,602 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:15,603 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:15,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:15,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef1bd73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:15,604 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:15,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:15,606 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:15,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f5cd6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:15,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:15,610 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:15,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1612488f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:15,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:15,613 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:15,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff1f1f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d78f05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:15,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:15,616 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ec01134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:15,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:15,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:15,619 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:15,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a081598, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6777ea8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:15,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:15,622 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@779a286f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:15,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:15,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54571dc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa51225, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:15,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:15,628 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c197857, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:15,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:15,631 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b66dceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a6dea6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:15,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:15,634 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:15,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ee9f900, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:15,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:15,636 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:15,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4e5c6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41c8eaf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:15,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:15,639 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1118bf09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:15,641 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:15,642 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:15,643 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339e1ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f2f8db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:15,644 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:15,645 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38dcce6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:15,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:15,646 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:15,648 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:15,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9c26bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:15,652 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:15,652 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:15,653 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:15,653 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:15,653 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:15,653 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:15,653 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:15,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:15,657 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:15,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:15,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:15,658 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:15,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:15,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:15,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:15,660 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:15,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:15,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:15,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:15,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:15,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:15,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57006, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:15,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:34:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:34:15,668 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:34:15,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:34:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:34:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:34:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:34:15,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:34:15,676 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:34:15,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-11T20:34:15,693 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T20:34:15,699 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:15,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:15,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:15,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:34:15,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:15,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:15,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:15,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:15,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:15,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:15,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T20:34:15,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733949255806, max timestamp=1733949255834Num uniques 859; 2024-12-11T20:34:15,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T20:34:15,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/7b28ebbe7c0e45c1932093042e45b19c is 50, key is test_row_0/A:col10/1733949255732/Put/seqid=0 2024-12-11T20:34:15,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949255806, max timestamp=1733949255834Num uniques 859; 2024-12-11T20:34:15,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:15,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949255806, max timestamp=1733949255836Num uniques 859; 2024-12-11T20:34:15,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:15,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-11T20:34:15,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:15,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741837_1013 (size=12001) 2024-12-11T20:34:15,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/7b28ebbe7c0e45c1932093042e45b19c 2024-12-11T20:34:15,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:15,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:15,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:15,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:15,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:15,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949315940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:15,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949315939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:15,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949315951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:15,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:15,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949315955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:15,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949315957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:15,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:15,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:15,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T20:34:16,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/48c8b56b0e6845c385cb31f03f9ac6af is 50, key is test_row_0/B:col10/1733949255732/Put/seqid=0 2024-12-11T20:34:16,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-11T20:34:16,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:16,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741838_1014 (size=12001) 2024-12-11T20:34:16,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:16,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:16,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:16,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:16,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/48c8b56b0e6845c385cb31f03f9ac6af 2024-12-11T20:34:16,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949316111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949316109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949316111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949316111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949316111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,126 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/d5b33b1feec74b23bdb97066111b5e6e is 50, key is test_row_0/C:col10/1733949255732/Put/seqid=0 2024-12-11T20:34:16,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741839_1015 (size=12001) 2024-12-11T20:34:16,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/d5b33b1feec74b23bdb97066111b5e6e 2024-12-11T20:34:16,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/7b28ebbe7c0e45c1932093042e45b19c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7b28ebbe7c0e45c1932093042e45b19c 2024-12-11T20:34:16,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7b28ebbe7c0e45c1932093042e45b19c, entries=150, sequenceid=12, filesize=11.7 K 2024-12-11T20:34:16,204 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-11T20:34:16,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/48c8b56b0e6845c385cb31f03f9ac6af as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/48c8b56b0e6845c385cb31f03f9ac6af 2024-12-11T20:34:16,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/48c8b56b0e6845c385cb31f03f9ac6af, entries=150, sequenceid=12, filesize=11.7 K 2024-12-11T20:34:16,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/d5b33b1feec74b23bdb97066111b5e6e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d5b33b1feec74b23bdb97066111b5e6e 2024-12-11T20:34:16,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d5b33b1feec74b23bdb97066111b5e6e, entries=150, sequenceid=12, filesize=11.7 K 2024-12-11T20:34:16,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for c0a19b15b87adee237235cecb1e4c99f in 509ms, sequenceid=12, compaction requested=false 2024-12-11T20:34:16,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-11T20:34:16,252 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-11T20:34:16,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:16,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:16,267 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-12-11T20:34:16,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:16,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:16,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/8efebfd0503b4522aec7116c3cb30207 is 50, key is test_row_0/A:col10/1733949255948/Put/seqid=0 2024-12-11T20:34:16,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741840_1016 (size=12001) 2024-12-11T20:34:16,311 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/8efebfd0503b4522aec7116c3cb30207 2024-12-11T20:34:16,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T20:34:16,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/6871865fc1384259965df21b1986ce11 is 50, key is test_row_0/B:col10/1733949255948/Put/seqid=0 2024-12-11T20:34:16,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:16,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:16,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949316350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949316353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741841_1017 (size=12001) 2024-12-11T20:34:16,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949316356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,371 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949316361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949316365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949316477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949316478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949316479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949316480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949316479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949316697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949316698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949316698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949316699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:16,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949316699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:16,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:16,772 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/6871865fc1384259965df21b1986ce11 2024-12-11T20:34:16,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/6e968282688a4016b4195a4ec9fa3dab is 50, key is test_row_0/C:col10/1733949255948/Put/seqid=0 2024-12-11T20:34:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T20:34:16,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741842_1018 (size=12001) 2024-12-11T20:34:16,848 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/6e968282688a4016b4195a4ec9fa3dab 2024-12-11T20:34:16,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/8efebfd0503b4522aec7116c3cb30207 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/8efebfd0503b4522aec7116c3cb30207 2024-12-11T20:34:16,891 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/8efebfd0503b4522aec7116c3cb30207, entries=150, sequenceid=48, filesize=11.7 K 2024-12-11T20:34:16,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/6871865fc1384259965df21b1986ce11 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6871865fc1384259965df21b1986ce11 2024-12-11T20:34:16,908 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6871865fc1384259965df21b1986ce11, entries=150, sequenceid=48, filesize=11.7 K 2024-12-11T20:34:16,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/6e968282688a4016b4195a4ec9fa3dab as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6e968282688a4016b4195a4ec9fa3dab 2024-12-11T20:34:16,929 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6e968282688a4016b4195a4ec9fa3dab, entries=150, sequenceid=48, filesize=11.7 K 2024-12-11T20:34:16,931 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=46.96 KB/48090 for c0a19b15b87adee237235cecb1e4c99f in 665ms, sequenceid=48, compaction requested=false 2024-12-11T20:34:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:16,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-11T20:34:16,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-11T20:34:16,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-11T20:34:16,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2380 sec 2024-12-11T20:34:16,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.2610 sec 2024-12-11T20:34:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:17,016 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T20:34:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:17,017 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:17,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/1531b5106fac48a6933ac88b02579d9f is 50, key is test_row_0/A:col10/1733949257011/Put/seqid=0 2024-12-11T20:34:17,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741843_1019 (size=14341) 2024-12-11T20:34:17,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/1531b5106fac48a6933ac88b02579d9f 2024-12-11T20:34:17,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/7ae950731eb64b6fbfe181024f8bf131 is 50, key is test_row_0/B:col10/1733949257011/Put/seqid=0 2024-12-11T20:34:17,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741844_1020 (size=12001) 2024-12-11T20:34:17,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733949257041, max timestamp=1733949257116Num uniques 859; 2024-12-11T20:34:17,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T20:34:17,120 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733949257041, max timestamp=1733949257116Num uniques 859; 2024-12-11T20:34:17,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T20:34:17,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733949257041, max timestamp=1733949257116Num uniques 859; 2024-12-11T20:34:17,123 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T20:34:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949317155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949317155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949317158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949317168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949317168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949317278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949317278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949317278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949317288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949317288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/7ae950731eb64b6fbfe181024f8bf131 2024-12-11T20:34:17,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949317487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949317488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,492 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/afd1bd108e3f4c7aa6564168ede4f6c0 is 50, key is test_row_0/C:col10/1733949257011/Put/seqid=0 2024-12-11T20:34:17,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949317497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949317497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,507 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949317498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741845_1021 (size=12001) 2024-12-11T20:34:17,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/afd1bd108e3f4c7aa6564168ede4f6c0 2024-12-11T20:34:17,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/1531b5106fac48a6933ac88b02579d9f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1531b5106fac48a6933ac88b02579d9f 2024-12-11T20:34:17,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1531b5106fac48a6933ac88b02579d9f, entries=200, sequenceid=59, filesize=14.0 K 2024-12-11T20:34:17,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/7ae950731eb64b6fbfe181024f8bf131 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ae950731eb64b6fbfe181024f8bf131 2024-12-11T20:34:17,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ae950731eb64b6fbfe181024f8bf131, entries=150, sequenceid=59, filesize=11.7 K 2024-12-11T20:34:17,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/afd1bd108e3f4c7aa6564168ede4f6c0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/afd1bd108e3f4c7aa6564168ede4f6c0 2024-12-11T20:34:17,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/afd1bd108e3f4c7aa6564168ede4f6c0, entries=150, sequenceid=59, filesize=11.7 K 2024-12-11T20:34:17,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c0a19b15b87adee237235cecb1e4c99f in 567ms, sequenceid=59, compaction requested=true 2024-12-11T20:34:17,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:17,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:17,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:17,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:17,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:17,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:17,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:17,590 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:17,590 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:17,594 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:17,594 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:17,596 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/B is initiating minor compaction (all files) 2024-12-11T20:34:17,597 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/B in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:17,598 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/48c8b56b0e6845c385cb31f03f9ac6af, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6871865fc1384259965df21b1986ce11, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ae950731eb64b6fbfe181024f8bf131] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=35.2 K 2024-12-11T20:34:17,599 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/A is initiating minor compaction (all files) 2024-12-11T20:34:17,599 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/A in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:17,599 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7b28ebbe7c0e45c1932093042e45b19c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/8efebfd0503b4522aec7116c3cb30207, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1531b5106fac48a6933ac88b02579d9f] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=37.4 K 2024-12-11T20:34:17,599 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 48c8b56b0e6845c385cb31f03f9ac6af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733949255696 2024-12-11T20:34:17,601 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6871865fc1384259965df21b1986ce11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733949255945 2024-12-11T20:34:17,602 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b28ebbe7c0e45c1932093042e45b19c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733949255696 2024-12-11T20:34:17,602 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ae950731eb64b6fbfe181024f8bf131, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733949256352 2024-12-11T20:34:17,603 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8efebfd0503b4522aec7116c3cb30207, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733949255945 2024-12-11T20:34:17,604 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1531b5106fac48a6933ac88b02579d9f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733949256334 2024-12-11T20:34:17,660 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#B#compaction#9 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:17,664 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/51de29dcb3324c01a0266ec03f3b5e85 is 50, key is test_row_0/B:col10/1733949257011/Put/seqid=0 2024-12-11T20:34:17,666 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#A#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:17,667 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/5b03928a41ca4bad9207a1c9e6955345 is 50, key is test_row_0/A:col10/1733949257011/Put/seqid=0 2024-12-11T20:34:17,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741847_1023 (size=12104) 2024-12-11T20:34:17,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741846_1022 (size=12104) 2024-12-11T20:34:17,734 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-11T20:34:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:17,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-11T20:34:17,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:17,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:17,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:17,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:17,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:17,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:17,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/393de5cae4684c04835e553b24cf56a3 is 50, key is test_row_0/A:col10/1733949257164/Put/seqid=0 2024-12-11T20:34:17,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949317824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949317828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949317828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741848_1024 (size=14341) 2024-12-11T20:34:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/393de5cae4684c04835e553b24cf56a3 2024-12-11T20:34:17,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-11T20:34:17,848 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:17,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,848 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:17,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949317838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-11T20:34:17,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T20:34:17,852 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:17,855 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:17,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:17,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949317838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/b44fbaf0f514458d968f2023ac6368b3 is 50, key is test_row_0/B:col10/1733949257164/Put/seqid=0 2024-12-11T20:34:17,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741849_1025 (size=12001) 2024-12-11T20:34:17,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/b44fbaf0f514458d968f2023ac6368b3 2024-12-11T20:34:17,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/1dbd509fcee3445bb214e6877cb78310 is 50, key is test_row_0/C:col10/1733949257164/Put/seqid=0 2024-12-11T20:34:17,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949317948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949317948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949317948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T20:34:17,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949317958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949317967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:17,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741850_1026 (size=12001) 2024-12-11T20:34:17,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/1dbd509fcee3445bb214e6877cb78310 2024-12-11T20:34:17,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/393de5cae4684c04835e553b24cf56a3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/393de5cae4684c04835e553b24cf56a3 2024-12-11T20:34:18,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/393de5cae4684c04835e553b24cf56a3, entries=200, sequenceid=95, filesize=14.0 K 2024-12-11T20:34:18,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T20:34:18,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:18,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/b44fbaf0f514458d968f2023ac6368b3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b44fbaf0f514458d968f2023ac6368b3 2024-12-11T20:34:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b44fbaf0f514458d968f2023ac6368b3, entries=150, sequenceid=95, filesize=11.7 K 2024-12-11T20:34:18,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/1dbd509fcee3445bb214e6877cb78310 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1dbd509fcee3445bb214e6877cb78310 2024-12-11T20:34:18,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1dbd509fcee3445bb214e6877cb78310, entries=150, sequenceid=95, filesize=11.7 K 2024-12-11T20:34:18,085 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=46.96 KB/48090 for c0a19b15b87adee237235cecb1e4c99f in 285ms, sequenceid=95, compaction requested=true 2024-12-11T20:34:18,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:18,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:18,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T20:34:18,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:18,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-11T20:34:18,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:18,086 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-11T20:34:18,125 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/5b03928a41ca4bad9207a1c9e6955345 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/5b03928a41ca4bad9207a1c9e6955345 2024-12-11T20:34:18,148 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/51de29dcb3324c01a0266ec03f3b5e85 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/51de29dcb3324c01a0266ec03f3b5e85 2024-12-11T20:34:18,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:18,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T20:34:18,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T20:34:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:18,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:18,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:18,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:18,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T20:34:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/cdcca54a28744e9cbf3d9f31abffbf51 is 50, key is test_row_0/A:col10/1733949258160/Put/seqid=0 2024-12-11T20:34:18,193 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/B of c0a19b15b87adee237235cecb1e4c99f into 51de29dcb3324c01a0266ec03f3b5e85(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:18,193 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:18,195 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/A of c0a19b15b87adee237235cecb1e4c99f into 5b03928a41ca4bad9207a1c9e6955345(size=11.8 K), total size for store is 25.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:18,195 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:18,198 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/A, priority=13, startTime=1733949257585; duration=0sec 2024-12-11T20:34:18,199 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/B, priority=13, startTime=1733949257588; duration=0sec 2024-12-11T20:34:18,199 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-11T20:34:18,200 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:18,200 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:18,200 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:18,201 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-11T20:34:18,202 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:18,202 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:18,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741851_1027 (size=12001) 2024-12-11T20:34:18,205 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:18,207 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-11T20:34:18,207 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-11T20:34:18,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/cdcca54a28744e9cbf3d9f31abffbf51 2024-12-11T20:34:18,207 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-11T20:34:18,207 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. because compaction request was cancelled 2024-12-11T20:34:18,208 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:18,208 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-11T20:34:18,207 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/C is initiating minor compaction (all files) 2024-12-11T20:34:18,208 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/C in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,208 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d5b33b1feec74b23bdb97066111b5e6e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6e968282688a4016b4195a4ec9fa3dab, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/afd1bd108e3f4c7aa6564168ede4f6c0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1dbd509fcee3445bb214e6877cb78310] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=46.9 K 2024-12-11T20:34:18,214 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-11T20:34:18,214 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-11T20:34:18,214 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. because compaction request was cancelled 2024-12-11T20:34:18,215 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:18,215 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-11T20:34:18,216 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5b33b1feec74b23bdb97066111b5e6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733949255696 2024-12-11T20:34:18,218 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-11T20:34:18,218 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-11T20:34:18,218 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. because compaction request was cancelled 2024-12-11T20:34:18,218 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:18,219 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e968282688a4016b4195a4ec9fa3dab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733949255945 2024-12-11T20:34:18,220 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting afd1bd108e3f4c7aa6564168ede4f6c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733949256352 2024-12-11T20:34:18,222 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dbd509fcee3445bb214e6877cb78310, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733949257142 2024-12-11T20:34:18,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/043b917cf31d40d4836187df87687fa8 is 50, key is test_row_0/B:col10/1733949258160/Put/seqid=0 2024-12-11T20:34:18,253 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#C#compaction#16 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:18,254 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/3d7bef8e38ab4aab888905eb86328239 is 50, key is test_row_0/C:col10/1733949257164/Put/seqid=0 2024-12-11T20:34:18,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949318252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949318254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949318258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949318267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741852_1028 (size=12001) 2024-12-11T20:34:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949318272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741853_1029 (size=12139) 2024-12-11T20:34:18,311 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/3d7bef8e38ab4aab888905eb86328239 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/3d7bef8e38ab4aab888905eb86328239 2024-12-11T20:34:18,324 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/C of c0a19b15b87adee237235cecb1e4c99f into 3d7bef8e38ab4aab888905eb86328239(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:18,324 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:18,324 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/C, priority=12, startTime=1733949258086; duration=0sec 2024-12-11T20:34:18,324 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:18,324 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:18,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T20:34:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949318377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949318378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949318389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,395 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949318397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949318397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,453 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:34:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T20:34:18,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T20:34:18,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:18,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,554 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-11T20:34:18,559 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-11T20:34:18,563 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T20:34:18,563 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-11T20:34:18,564 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-11T20:34:18,564 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-11T20:34:18,565 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-11T20:34:18,565 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-11T20:34:18,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949318587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949318588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949318607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949318607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949318607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T20:34:18,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:18,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/043b917cf31d40d4836187df87687fa8 2024-12-11T20:34:18,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/db2a7804a33c43cbb0ef79f430316b59 is 50, key is test_row_0/C:col10/1733949258160/Put/seqid=0 2024-12-11T20:34:18,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741854_1030 (size=12001) 2024-12-11T20:34:18,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T20:34:18,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:18,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949318897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949318897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949318918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949318918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949318918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:18,923 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:18,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T20:34:18,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:18,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:18,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T20:34:19,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T20:34:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:19,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:19,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:19,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/db2a7804a33c43cbb0ef79f430316b59 2024-12-11T20:34:19,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/cdcca54a28744e9cbf3d9f31abffbf51 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/cdcca54a28744e9cbf3d9f31abffbf51 2024-12-11T20:34:19,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/cdcca54a28744e9cbf3d9f31abffbf51, entries=150, sequenceid=108, filesize=11.7 K 2024-12-11T20:34:19,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/043b917cf31d40d4836187df87687fa8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/043b917cf31d40d4836187df87687fa8 2024-12-11T20:34:19,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/043b917cf31d40d4836187df87687fa8, entries=150, sequenceid=108, filesize=11.7 K 2024-12-11T20:34:19,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/db2a7804a33c43cbb0ef79f430316b59 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/db2a7804a33c43cbb0ef79f430316b59 2024-12-11T20:34:19,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/db2a7804a33c43cbb0ef79f430316b59, entries=150, sequenceid=108, filesize=11.7 K 2024-12-11T20:34:19,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c0a19b15b87adee237235cecb1e4c99f in 1036ms, sequenceid=108, compaction requested=true 2024-12-11T20:34:19,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:19,201 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:19,201 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:19,203 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:19,203 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/B is initiating minor compaction (all files) 2024-12-11T20:34:19,203 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/B in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:19,204 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/51de29dcb3324c01a0266ec03f3b5e85, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b44fbaf0f514458d968f2023ac6368b3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/043b917cf31d40d4836187df87687fa8] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=35.3 K 2024-12-11T20:34:19,204 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:19,204 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/A is initiating minor compaction (all files) 2024-12-11T20:34:19,206 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/A in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:19,206 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/5b03928a41ca4bad9207a1c9e6955345, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/393de5cae4684c04835e553b24cf56a3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/cdcca54a28744e9cbf3d9f31abffbf51] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=37.5 K 2024-12-11T20:34:19,207 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 51de29dcb3324c01a0266ec03f3b5e85, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733949256352 2024-12-11T20:34:19,208 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b03928a41ca4bad9207a1c9e6955345, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733949256352 2024-12-11T20:34:19,209 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting b44fbaf0f514458d968f2023ac6368b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733949257142 2024-12-11T20:34:19,209 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 393de5cae4684c04835e553b24cf56a3, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733949257142 2024-12-11T20:34:19,210 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdcca54a28744e9cbf3d9f31abffbf51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733949257826 2024-12-11T20:34:19,210 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 043b917cf31d40d4836187df87687fa8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733949257826 2024-12-11T20:34:19,234 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#B#compaction#18 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:19,235 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/3127b3de0fc04f3badf9d50245713167 is 50, key is test_row_0/B:col10/1733949258160/Put/seqid=0 2024-12-11T20:34:19,245 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:19,246 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/01dfcdd03b7d4f5a8368a4825d2805e8 is 50, key is test_row_0/A:col10/1733949258160/Put/seqid=0 2024-12-11T20:34:19,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741855_1031 (size=12207) 2024-12-11T20:34:19,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-11T20:34:19,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:19,274 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T20:34:19,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:19,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:19,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:19,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:19,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:19,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:19,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741856_1032 (size=12207) 2024-12-11T20:34:19,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/76082bcb39b047aa93b1f36b2bda8ee7 is 50, key is test_row_0/A:col10/1733949258264/Put/seqid=0 2024-12-11T20:34:19,299 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/01dfcdd03b7d4f5a8368a4825d2805e8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/01dfcdd03b7d4f5a8368a4825d2805e8 2024-12-11T20:34:19,319 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/A of c0a19b15b87adee237235cecb1e4c99f into 01dfcdd03b7d4f5a8368a4825d2805e8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:19,320 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:19,320 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/A, priority=13, startTime=1733949259201; duration=0sec 2024-12-11T20:34:19,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741857_1033 (size=12051) 2024-12-11T20:34:19,323 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/76082bcb39b047aa93b1f36b2bda8ee7 2024-12-11T20:34:19,323 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:19,324 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:19,324 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-11T20:34:19,338 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-11T20:34:19,339 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-11T20:34:19,339 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. because compaction request was cancelled 2024-12-11T20:34:19,339 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:19,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/35e3ff6660594b3e8561e4ff2c3efefc is 50, key is test_row_0/B:col10/1733949258264/Put/seqid=0 2024-12-11T20:34:19,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741858_1034 (size=12051) 2024-12-11T20:34:19,358 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/35e3ff6660594b3e8561e4ff2c3efefc 2024-12-11T20:34:19,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/aac9b1ac6b6b4f49b6620a8cfcd90f46 is 50, key is test_row_0/C:col10/1733949258264/Put/seqid=0 2024-12-11T20:34:19,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741859_1035 (size=12051) 2024-12-11T20:34:19,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:19,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949319428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949319431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949319437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949319437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949319437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,441 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949319547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949319547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949319548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949319548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,664 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/3127b3de0fc04f3badf9d50245713167 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/3127b3de0fc04f3badf9d50245713167 2024-12-11T20:34:19,678 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/B of c0a19b15b87adee237235cecb1e4c99f into 3127b3de0fc04f3badf9d50245713167(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:19,678 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:19,678 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/B, priority=13, startTime=1733949259201; duration=0sec 2024-12-11T20:34:19,679 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:19,679 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:19,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949319756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949319757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949319757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949319757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:19,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:19,790 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/aac9b1ac6b6b4f49b6620a8cfcd90f46 2024-12-11T20:34:19,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/76082bcb39b047aa93b1f36b2bda8ee7 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/76082bcb39b047aa93b1f36b2bda8ee7 2024-12-11T20:34:19,810 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/76082bcb39b047aa93b1f36b2bda8ee7, entries=150, sequenceid=131, filesize=11.8 K 2024-12-11T20:34:19,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/35e3ff6660594b3e8561e4ff2c3efefc as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/35e3ff6660594b3e8561e4ff2c3efefc 2024-12-11T20:34:19,822 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/35e3ff6660594b3e8561e4ff2c3efefc, entries=150, sequenceid=131, filesize=11.8 K 2024-12-11T20:34:19,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/aac9b1ac6b6b4f49b6620a8cfcd90f46 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/aac9b1ac6b6b4f49b6620a8cfcd90f46 2024-12-11T20:34:19,841 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/aac9b1ac6b6b4f49b6620a8cfcd90f46, entries=150, sequenceid=131, filesize=11.8 K 2024-12-11T20:34:19,844 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for c0a19b15b87adee237235cecb1e4c99f in 569ms, sequenceid=131, compaction requested=true 2024-12-11T20:34:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-11T20:34:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-11T20:34:19,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-11T20:34:19,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9920 sec 2024-12-11T20:34:19,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.0060 sec 2024-12-11T20:34:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-11T20:34:19,998 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:19,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-11T20:34:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T20:34:20,002 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:20,003 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:20,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:20,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T20:34:20,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:20,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:20,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:20,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:20,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:20,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:20,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/328ac0fcd0cd4418b03ad1ece7c27ca6 is 50, key is test_row_0/A:col10/1733949259432/Put/seqid=0 2024-12-11T20:34:20,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T20:34:20,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949320107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741860_1036 (size=12151) 2024-12-11T20:34:20,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/328ac0fcd0cd4418b03ad1ece7c27ca6 2024-12-11T20:34:20,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949320114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949320115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949320118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/f4c73fb614ee4e45b4f2a672559142b7 is 50, key is test_row_0/B:col10/1733949259432/Put/seqid=0 2024-12-11T20:34:20,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741861_1037 (size=12151) 2024-12-11T20:34:20,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T20:34:20,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:20,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949320227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949320238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949320238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949320238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T20:34:20,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T20:34:20,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949320437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949320448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949320448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949320449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949320449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,455 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_0' in c0a19b15b87adee237235cecb1e4c99f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T20:34:20,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:20,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/f4c73fb614ee4e45b4f2a672559142b7 2024-12-11T20:34:20,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/d1133f81671b4c78b7f43e64b4acda20 is 50, key is test_row_0/C:col10/1733949259432/Put/seqid=0 2024-12-11T20:34:20,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741862_1038 (size=12151) 2024-12-11T20:34:20,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T20:34:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T20:34:20,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949320748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949320757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949320758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949320758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:20,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T20:34:20,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:20,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T20:34:20,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:20,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:20,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:20,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/d1133f81671b4c78b7f43e64b4acda20 2024-12-11T20:34:20,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/328ac0fcd0cd4418b03ad1ece7c27ca6 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/328ac0fcd0cd4418b03ad1ece7c27ca6 2024-12-11T20:34:21,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/328ac0fcd0cd4418b03ad1ece7c27ca6, entries=150, sequenceid=149, filesize=11.9 K 2024-12-11T20:34:21,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/f4c73fb614ee4e45b4f2a672559142b7 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f4c73fb614ee4e45b4f2a672559142b7 2024-12-11T20:34:21,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f4c73fb614ee4e45b4f2a672559142b7, entries=150, sequenceid=149, filesize=11.9 K 2024-12-11T20:34:21,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/d1133f81671b4c78b7f43e64b4acda20 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d1133f81671b4c78b7f43e64b4acda20 2024-12-11T20:34:21,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d1133f81671b4c78b7f43e64b4acda20, entries=150, sequenceid=149, filesize=11.9 K 2024-12-11T20:34:21,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0a19b15b87adee237235cecb1e4c99f in 971ms, sequenceid=149, compaction requested=true 2024-12-11T20:34:21,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:21,043 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:21,043 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:21,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:21,044 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:21,045 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:21,045 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/A is initiating minor compaction (all files) 2024-12-11T20:34:21,045 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/B is initiating minor compaction (all files) 2024-12-11T20:34:21,045 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/A in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:21,045 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/B in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:21,045 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/3127b3de0fc04f3badf9d50245713167, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/35e3ff6660594b3e8561e4ff2c3efefc, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f4c73fb614ee4e45b4f2a672559142b7] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=35.6 K 2024-12-11T20:34:21,045 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/01dfcdd03b7d4f5a8368a4825d2805e8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/76082bcb39b047aa93b1f36b2bda8ee7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/328ac0fcd0cd4418b03ad1ece7c27ca6] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=35.6 K 2024-12-11T20:34:21,046 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3127b3de0fc04f3badf9d50245713167, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733949257826 2024-12-11T20:34:21,046 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01dfcdd03b7d4f5a8368a4825d2805e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733949257826 2024-12-11T20:34:21,046 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 35e3ff6660594b3e8561e4ff2c3efefc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733949258254 2024-12-11T20:34:21,047 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76082bcb39b047aa93b1f36b2bda8ee7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733949258254 2024-12-11T20:34:21,047 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f4c73fb614ee4e45b4f2a672559142b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733949259432 2024-12-11T20:34:21,050 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 328ac0fcd0cd4418b03ad1ece7c27ca6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733949259432 2024-12-11T20:34:21,065 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#B#compaction#26 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:21,065 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/dc704688418a4408876304a0259d1314 is 50, key is test_row_0/B:col10/1733949259432/Put/seqid=0 2024-12-11T20:34:21,073 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#A#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:21,074 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/c05940b3ebda423ca6618949787e1785 is 50, key is test_row_0/A:col10/1733949259432/Put/seqid=0 2024-12-11T20:34:21,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-11T20:34:21,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:21,090 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:34:21,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:21,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:21,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:21,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:21,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:21,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:21,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741864_1040 (size=12459) 2024-12-11T20:34:21,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741863_1039 (size=12459) 2024-12-11T20:34:21,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/70bbb5a438a24d7a822cee14b9986f8c is 50, key is test_row_0/A:col10/1733949260115/Put/seqid=0 2024-12-11T20:34:21,120 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/c05940b3ebda423ca6618949787e1785 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/c05940b3ebda423ca6618949787e1785 2024-12-11T20:34:21,121 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/dc704688418a4408876304a0259d1314 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/dc704688418a4408876304a0259d1314 2024-12-11T20:34:21,134 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/A of c0a19b15b87adee237235cecb1e4c99f into c05940b3ebda423ca6618949787e1785(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:21,134 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/B of c0a19b15b87adee237235cecb1e4c99f into dc704688418a4408876304a0259d1314(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:21,134 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:21,134 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:21,134 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/A, priority=13, startTime=1733949261042; duration=0sec 2024-12-11T20:34:21,134 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/B, priority=13, startTime=1733949261043; duration=0sec 2024-12-11T20:34:21,134 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:21,134 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:21,134 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:21,137 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:21,137 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:21,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T20:34:21,140 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:21,140 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/C is initiating minor compaction (all files) 2024-12-11T20:34:21,140 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/C in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:21,141 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/3d7bef8e38ab4aab888905eb86328239, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/db2a7804a33c43cbb0ef79f430316b59, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/aac9b1ac6b6b4f49b6620a8cfcd90f46, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d1133f81671b4c78b7f43e64b4acda20] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=47.2 K 2024-12-11T20:34:21,142 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d7bef8e38ab4aab888905eb86328239, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733949257142 2024-12-11T20:34:21,143 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting db2a7804a33c43cbb0ef79f430316b59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733949257826 2024-12-11T20:34:21,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741865_1041 (size=12151) 2024-12-11T20:34:21,146 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting aac9b1ac6b6b4f49b6620a8cfcd90f46, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1733949258254 2024-12-11T20:34:21,147 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1133f81671b4c78b7f43e64b4acda20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733949259432 2024-12-11T20:34:21,176 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#C#compaction#29 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:21,176 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/9b8fd8b2110b4024bf54b244513e4a2e is 50, key is test_row_0/C:col10/1733949259432/Put/seqid=0 2024-12-11T20:34:21,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741866_1042 (size=12425) 2024-12-11T20:34:21,197 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/9b8fd8b2110b4024bf54b244513e4a2e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/9b8fd8b2110b4024bf54b244513e4a2e 2024-12-11T20:34:21,209 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/C of c0a19b15b87adee237235cecb1e4c99f into 9b8fd8b2110b4024bf54b244513e4a2e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:21,209 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:21,209 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/C, priority=12, startTime=1733949261043; duration=0sec 2024-12-11T20:34:21,210 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:21,210 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:21,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:21,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949321284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949321286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949321287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949321287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949321397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949321397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949321397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949321398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,545 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/70bbb5a438a24d7a822cee14b9986f8c 2024-12-11T20:34:21,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/bfef54fa90aa48a7836fbffb4699271a is 50, key is test_row_0/B:col10/1733949260115/Put/seqid=0 2024-12-11T20:34:21,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741867_1043 (size=12151) 2024-12-11T20:34:21,597 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/bfef54fa90aa48a7836fbffb4699271a 2024-12-11T20:34:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949321607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949321608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949321608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949321608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/6d18bf8718a04027ab4f52402586285c is 50, key is test_row_0/C:col10/1733949260115/Put/seqid=0 2024-12-11T20:34:21,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741868_1044 (size=12151) 2024-12-11T20:34:21,646 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/6d18bf8718a04027ab4f52402586285c 2024-12-11T20:34:21,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/70bbb5a438a24d7a822cee14b9986f8c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/70bbb5a438a24d7a822cee14b9986f8c 2024-12-11T20:34:21,671 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/70bbb5a438a24d7a822cee14b9986f8c, entries=150, sequenceid=165, filesize=11.9 K 2024-12-11T20:34:21,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/bfef54fa90aa48a7836fbffb4699271a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bfef54fa90aa48a7836fbffb4699271a 2024-12-11T20:34:21,682 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bfef54fa90aa48a7836fbffb4699271a, entries=150, sequenceid=165, filesize=11.9 K 2024-12-11T20:34:21,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/6d18bf8718a04027ab4f52402586285c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6d18bf8718a04027ab4f52402586285c 2024-12-11T20:34:21,698 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6d18bf8718a04027ab4f52402586285c, entries=150, sequenceid=165, filesize=11.9 K 2024-12-11T20:34:21,700 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0a19b15b87adee237235cecb1e4c99f in 610ms, sequenceid=165, compaction requested=false 2024-12-11T20:34:21,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:21,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:21,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-11T20:34:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-11T20:34:21,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-11T20:34:21,707 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7000 sec 2024-12-11T20:34:21,713 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.7120 sec 2024-12-11T20:34:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:21,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T20:34:21,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:21,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:21,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:21,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:21,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:21,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:21,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/ad88b8dbd9d84f71a2073a2088c0896c is 50, key is test_row_0/A:col10/1733949261918/Put/seqid=0 2024-12-11T20:34:21,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949321944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949321945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949321945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949321948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:21,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:21,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741869_1045 (size=16931) 2024-12-11T20:34:21,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/ad88b8dbd9d84f71a2073a2088c0896c 2024-12-11T20:34:21,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/33478ca0a3a44dbebfff6e45e3a55e41 is 50, key is test_row_0/B:col10/1733949261918/Put/seqid=0 2024-12-11T20:34:22,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741870_1046 (size=12151) 2024-12-11T20:34:22,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/33478ca0a3a44dbebfff6e45e3a55e41 2024-12-11T20:34:22,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/324c392b35544801ac3c59725361de22 is 50, key is test_row_0/C:col10/1733949261918/Put/seqid=0 2024-12-11T20:34:22,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741871_1047 (size=12151) 2024-12-11T20:34:22,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/324c392b35544801ac3c59725361de22 2024-12-11T20:34:22,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949322067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949322067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949322067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949322068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/ad88b8dbd9d84f71a2073a2088c0896c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/ad88b8dbd9d84f71a2073a2088c0896c 2024-12-11T20:34:22,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/ad88b8dbd9d84f71a2073a2088c0896c, entries=250, sequenceid=186, filesize=16.5 K 2024-12-11T20:34:22,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/33478ca0a3a44dbebfff6e45e3a55e41 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/33478ca0a3a44dbebfff6e45e3a55e41 2024-12-11T20:34:22,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/33478ca0a3a44dbebfff6e45e3a55e41, entries=150, sequenceid=186, filesize=11.9 K 2024-12-11T20:34:22,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/324c392b35544801ac3c59725361de22 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/324c392b35544801ac3c59725361de22 2024-12-11T20:34:22,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/324c392b35544801ac3c59725361de22, entries=150, sequenceid=186, filesize=11.9 K 2024-12-11T20:34:22,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c0a19b15b87adee237235cecb1e4c99f in 192ms, sequenceid=186, compaction requested=true 2024-12-11T20:34:22,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:22,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:22,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:22,111 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:22,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:22,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:22,112 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:22,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:22,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:22,113 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:22,113 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/A is initiating minor compaction (all files) 2024-12-11T20:34:22,113 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/A in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:22,113 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:22,113 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/c05940b3ebda423ca6618949787e1785, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/70bbb5a438a24d7a822cee14b9986f8c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/ad88b8dbd9d84f71a2073a2088c0896c] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=40.6 K 2024-12-11T20:34:22,113 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/B is initiating minor compaction (all files) 2024-12-11T20:34:22,113 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/B in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:22,114 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/dc704688418a4408876304a0259d1314, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bfef54fa90aa48a7836fbffb4699271a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/33478ca0a3a44dbebfff6e45e3a55e41] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=35.9 K 2024-12-11T20:34:22,114 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting c05940b3ebda423ca6618949787e1785, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733949259432 2024-12-11T20:34:22,115 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70bbb5a438a24d7a822cee14b9986f8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733949260110 2024-12-11T20:34:22,115 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting dc704688418a4408876304a0259d1314, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733949259432 2024-12-11T20:34:22,115 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad88b8dbd9d84f71a2073a2088c0896c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733949261282 2024-12-11T20:34:22,115 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting bfef54fa90aa48a7836fbffb4699271a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733949260110 2024-12-11T20:34:22,116 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 33478ca0a3a44dbebfff6e45e3a55e41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733949261285 2024-12-11T20:34:22,129 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#B#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:22,130 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/d9a3bcc7805e4e928bb2eb4640942afb is 50, key is test_row_0/B:col10/1733949261918/Put/seqid=0 2024-12-11T20:34:22,131 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:22,132 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/18c82d0a12b846c19ba2ec66f3acd28b is 50, key is test_row_0/A:col10/1733949261918/Put/seqid=0 2024-12-11T20:34:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-11T20:34:22,147 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:22,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-11T20:34:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T20:34:22,150 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:22,151 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:22,152 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:22,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741872_1048 (size=12561) 2024-12-11T20:34:22,183 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/d9a3bcc7805e4e928bb2eb4640942afb as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d9a3bcc7805e4e928bb2eb4640942afb 2024-12-11T20:34:22,193 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/B of c0a19b15b87adee237235cecb1e4c99f into d9a3bcc7805e4e928bb2eb4640942afb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:22,193 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:22,193 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/B, priority=13, startTime=1733949262111; duration=0sec 2024-12-11T20:34:22,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741873_1049 (size=12561) 2024-12-11T20:34:22,194 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:22,194 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:22,194 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:22,195 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:22,196 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/C is initiating minor compaction (all files) 2024-12-11T20:34:22,196 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/C in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:22,196 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/9b8fd8b2110b4024bf54b244513e4a2e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6d18bf8718a04027ab4f52402586285c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/324c392b35544801ac3c59725361de22] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=35.9 K 2024-12-11T20:34:22,198 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b8fd8b2110b4024bf54b244513e4a2e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1733949259432 2024-12-11T20:34:22,200 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d18bf8718a04027ab4f52402586285c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733949260110 2024-12-11T20:34:22,202 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 324c392b35544801ac3c59725361de22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733949261285 2024-12-11T20:34:22,204 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/18c82d0a12b846c19ba2ec66f3acd28b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/18c82d0a12b846c19ba2ec66f3acd28b 2024-12-11T20:34:22,216 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/A of c0a19b15b87adee237235cecb1e4c99f into 18c82d0a12b846c19ba2ec66f3acd28b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:22,216 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:22,216 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/A, priority=13, startTime=1733949262111; duration=0sec 2024-12-11T20:34:22,216 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:22,216 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:22,218 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#C#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:22,219 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/f88ccef3c1e948149d28700fda235bd4 is 50, key is test_row_0/C:col10/1733949261918/Put/seqid=0 2024-12-11T20:34:22,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741874_1050 (size=12527) 2024-12-11T20:34:22,236 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/f88ccef3c1e948149d28700fda235bd4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/f88ccef3c1e948149d28700fda235bd4 2024-12-11T20:34:22,246 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/C of c0a19b15b87adee237235cecb1e4c99f into f88ccef3c1e948149d28700fda235bd4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:22,246 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:22,246 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/C, priority=13, startTime=1733949262112; duration=0sec 2024-12-11T20:34:22,247 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:22,247 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T20:34:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:22,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:34:22,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:22,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:22,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/d00563dd3a0744a4b88b3be1e4dac023 is 50, key is test_row_0/A:col10/1733949262278/Put/seqid=0 2024-12-11T20:34:22,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741875_1051 (size=12151) 2024-12-11T20:34:22,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/d00563dd3a0744a4b88b3be1e4dac023 2024-12-11T20:34:22,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949322296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949322300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949322302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949322305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-11T20:34:22,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:22,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:22,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:22,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:22,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:22,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/a2467b7b59c94b00a1bc9bd5bfe9b32e is 50, key is test_row_0/B:col10/1733949262278/Put/seqid=0 2024-12-11T20:34:22,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741876_1052 (size=12151) 2024-12-11T20:34:22,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/a2467b7b59c94b00a1bc9bd5bfe9b32e 2024-12-11T20:34:22,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/87b67812b3274ba3b6031ab26baba174 is 50, key is test_row_0/C:col10/1733949262278/Put/seqid=0 2024-12-11T20:34:22,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741877_1053 (size=12151) 2024-12-11T20:34:22,393 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/87b67812b3274ba3b6031ab26baba174 2024-12-11T20:34:22,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/d00563dd3a0744a4b88b3be1e4dac023 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/d00563dd3a0744a4b88b3be1e4dac023 2024-12-11T20:34:22,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949322407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949322407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/d00563dd3a0744a4b88b3be1e4dac023, entries=150, sequenceid=205, filesize=11.9 K 2024-12-11T20:34:22,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/a2467b7b59c94b00a1bc9bd5bfe9b32e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/a2467b7b59c94b00a1bc9bd5bfe9b32e 2024-12-11T20:34:22,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949322417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949322418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/a2467b7b59c94b00a1bc9bd5bfe9b32e, entries=150, sequenceid=205, filesize=11.9 K 2024-12-11T20:34:22,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/87b67812b3274ba3b6031ab26baba174 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/87b67812b3274ba3b6031ab26baba174 2024-12-11T20:34:22,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/87b67812b3274ba3b6031ab26baba174, entries=150, sequenceid=205, filesize=11.9 K 2024-12-11T20:34:22,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0a19b15b87adee237235cecb1e4c99f in 163ms, sequenceid=205, compaction requested=false 2024-12-11T20:34:22,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:22,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-11T20:34:22,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:22,463 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:34:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:22,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T20:34:22,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/43afacfc9cca43c390d76326c6c55936 is 50, key is test_row_0/A:col10/1733949262294/Put/seqid=0 2024-12-11T20:34:22,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:22,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741878_1054 (size=12151) 2024-12-11T20:34:22,523 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/43afacfc9cca43c390d76326c6c55936 2024-12-11T20:34:22,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/0b2e1f47777b458296b77f2645f05c81 is 50, key is test_row_0/B:col10/1733949262294/Put/seqid=0 2024-12-11T20:34:22,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949322576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741879_1055 (size=12151) 2024-12-11T20:34:22,589 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/0b2e1f47777b458296b77f2645f05c81 2024-12-11T20:34:22,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/2a9209ce12264f1699c488d6ed6b566b is 50, key is test_row_0/C:col10/1733949262294/Put/seqid=0 2024-12-11T20:34:22,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949322617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949322618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949322627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949322627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741880_1056 (size=12151) 2024-12-11T20:34:22,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949322687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T20:34:22,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949322897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949322927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949322927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949322937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:22,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949322938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:22,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:23,033 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/2a9209ce12264f1699c488d6ed6b566b 2024-12-11T20:34:23,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/43afacfc9cca43c390d76326c6c55936 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/43afacfc9cca43c390d76326c6c55936 2024-12-11T20:34:23,062 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/43afacfc9cca43c390d76326c6c55936, entries=150, sequenceid=221, filesize=11.9 K 2024-12-11T20:34:23,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/0b2e1f47777b458296b77f2645f05c81 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/0b2e1f47777b458296b77f2645f05c81 2024-12-11T20:34:23,074 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/0b2e1f47777b458296b77f2645f05c81, entries=150, sequenceid=221, filesize=11.9 K 2024-12-11T20:34:23,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/2a9209ce12264f1699c488d6ed6b566b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/2a9209ce12264f1699c488d6ed6b566b 2024-12-11T20:34:23,088 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/2a9209ce12264f1699c488d6ed6b566b, entries=150, sequenceid=221, filesize=11.9 K 2024-12-11T20:34:23,089 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0a19b15b87adee237235cecb1e4c99f in 626ms, sequenceid=221, compaction requested=true 2024-12-11T20:34:23,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:23,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:23,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-11T20:34:23,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-11T20:34:23,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-11T20:34:23,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 941 msec 2024-12-11T20:34:23,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 953 msec 2024-12-11T20:34:23,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:23,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T20:34:23,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:23,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:23,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:23,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:23,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:23,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:23,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/bdf5e78781c34a53b295901984fe574a is 50, key is test_row_0/A:col10/1733949263208/Put/seqid=0 2024-12-11T20:34:23,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741881_1057 (size=12151) 2024-12-11T20:34:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-11T20:34:23,288 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:23,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-11T20:34:23,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T20:34:23,292 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:23,294 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:23,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:23,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:23,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949323336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:23,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T20:34:23,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:23,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949323438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:23,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949323438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:23,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T20:34:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:23,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949323448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949323448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:23,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949323449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:23,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T20:34:23,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:23,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:23,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:23,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T20:34:23,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/bdf5e78781c34a53b295901984fe574a 2024-12-11T20:34:23,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:23,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949323668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:23,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/b7af5517a7014912bc237289b0b4e82a is 50, key is test_row_0/B:col10/1733949263208/Put/seqid=0 2024-12-11T20:34:23,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741882_1058 (size=12151) 2024-12-11T20:34:23,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/b7af5517a7014912bc237289b0b4e82a 2024-12-11T20:34:23,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/d18795d22ef044b598a5274325339994 is 50, key is test_row_0/C:col10/1733949263208/Put/seqid=0 2024-12-11T20:34:23,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T20:34:23,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:23,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:23,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:23,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741883_1059 (size=12151) 2024-12-11T20:34:23,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T20:34:23,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T20:34:23,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:23,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:23,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:23,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:23,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:23,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949323987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:23,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T20:34:24,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:24,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:24,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:24,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:24,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:24,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:24,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/d18795d22ef044b598a5274325339994 2024-12-11T20:34:24,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/bdf5e78781c34a53b295901984fe574a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/bdf5e78781c34a53b295901984fe574a 2024-12-11T20:34:24,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T20:34:24,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:24,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:24,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:24,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:24,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:24,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:24,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/bdf5e78781c34a53b295901984fe574a, entries=150, sequenceid=238, filesize=11.9 K 2024-12-11T20:34:24,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/b7af5517a7014912bc237289b0b4e82a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b7af5517a7014912bc237289b0b4e82a 2024-12-11T20:34:24,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b7af5517a7014912bc237289b0b4e82a, entries=150, sequenceid=238, filesize=11.9 K 2024-12-11T20:34:24,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/d18795d22ef044b598a5274325339994 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d18795d22ef044b598a5274325339994 2024-12-11T20:34:24,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d18795d22ef044b598a5274325339994, entries=150, sequenceid=238, filesize=11.9 K 2024-12-11T20:34:24,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c0a19b15b87adee237235cecb1e4c99f in 1063ms, sequenceid=238, compaction requested=true 2024-12-11T20:34:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:24,274 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:24,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T20:34:24,274 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:24,276 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:24,276 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/B is initiating minor compaction (all files) 2024-12-11T20:34:24,276 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/B in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:24,276 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d9a3bcc7805e4e928bb2eb4640942afb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/a2467b7b59c94b00a1bc9bd5bfe9b32e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/0b2e1f47777b458296b77f2645f05c81, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b7af5517a7014912bc237289b0b4e82a] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=47.9 K 2024-12-11T20:34:24,277 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:24,277 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/A is initiating minor compaction (all files) 2024-12-11T20:34:24,278 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/A in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:24,278 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/18c82d0a12b846c19ba2ec66f3acd28b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/d00563dd3a0744a4b88b3be1e4dac023, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/43afacfc9cca43c390d76326c6c55936, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/bdf5e78781c34a53b295901984fe574a] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=47.9 K 2024-12-11T20:34:24,278 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d9a3bcc7805e4e928bb2eb4640942afb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733949261285 2024-12-11T20:34:24,278 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18c82d0a12b846c19ba2ec66f3acd28b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733949261285 2024-12-11T20:34:24,279 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting a2467b7b59c94b00a1bc9bd5bfe9b32e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733949261932 2024-12-11T20:34:24,279 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting d00563dd3a0744a4b88b3be1e4dac023, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733949261932 2024-12-11T20:34:24,281 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b2e1f47777b458296b77f2645f05c81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733949262294 2024-12-11T20:34:24,281 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43afacfc9cca43c390d76326c6c55936, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733949262294 2024-12-11T20:34:24,281 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting b7af5517a7014912bc237289b0b4e82a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733949262563 2024-12-11T20:34:24,282 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdf5e78781c34a53b295901984fe574a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733949262563 2024-12-11T20:34:24,313 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#B#compaction#47 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:24,314 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/b5878fde768e4becaed9ae662ba216ff is 50, key is test_row_0/B:col10/1733949263208/Put/seqid=0 2024-12-11T20:34:24,317 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#A#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:24,318 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/9128ac15b759492186b34a93d6bd4eb6 is 50, key is test_row_0/A:col10/1733949263208/Put/seqid=0 2024-12-11T20:34:24,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741884_1060 (size=12697) 2024-12-11T20:34:24,351 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/b5878fde768e4becaed9ae662ba216ff as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b5878fde768e4becaed9ae662ba216ff 2024-12-11T20:34:24,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741885_1061 (size=12697) 2024-12-11T20:34:24,369 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/B of c0a19b15b87adee237235cecb1e4c99f into b5878fde768e4becaed9ae662ba216ff(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:24,369 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:24,370 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/B, priority=12, startTime=1733949264274; duration=0sec 2024-12-11T20:34:24,370 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:24,370 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:24,370 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:24,372 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:24,373 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/C is initiating minor compaction (all files) 2024-12-11T20:34:24,373 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/C in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:24,373 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/f88ccef3c1e948149d28700fda235bd4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/87b67812b3274ba3b6031ab26baba174, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/2a9209ce12264f1699c488d6ed6b566b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d18795d22ef044b598a5274325339994] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=47.8 K 2024-12-11T20:34:24,374 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f88ccef3c1e948149d28700fda235bd4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1733949261285 2024-12-11T20:34:24,379 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 87b67812b3274ba3b6031ab26baba174, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1733949261932 2024-12-11T20:34:24,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-11T20:34:24,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:24,389 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T20:34:24,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:24,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:24,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:24,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:24,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:24,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:24,391 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a9209ce12264f1699c488d6ed6b566b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1733949262294 2024-12-11T20:34:24,393 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d18795d22ef044b598a5274325339994, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733949262563 2024-12-11T20:34:24,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/228f67d04821478fab6490ea59836246 is 50, key is test_row_0/A:col10/1733949263328/Put/seqid=0 2024-12-11T20:34:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T20:34:24,432 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:24,433 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/961357cb33c54703bda2fcf2da0ea372 is 50, key is test_row_0/C:col10/1733949263208/Put/seqid=0 2024-12-11T20:34:24,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741886_1062 (size=12151) 2024-12-11T20:34:24,435 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/228f67d04821478fab6490ea59836246 2024-12-11T20:34:24,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:24,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/d633b9a20f44491cbc8cb819c99b853c is 50, key is test_row_0/B:col10/1733949263328/Put/seqid=0 2024-12-11T20:34:24,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741887_1063 (size=12663) 2024-12-11T20:34:24,484 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/961357cb33c54703bda2fcf2da0ea372 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/961357cb33c54703bda2fcf2da0ea372 2024-12-11T20:34:24,501 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/C of c0a19b15b87adee237235cecb1e4c99f into 961357cb33c54703bda2fcf2da0ea372(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:24,501 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:24,501 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/C, priority=12, startTime=1733949264274; duration=0sec 2024-12-11T20:34:24,501 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:24,501 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:24,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949324499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741888_1064 (size=12151) 2024-12-11T20:34:24,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949324501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949324503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949324504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949324511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949324617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949324628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949324628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949324628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949324628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,783 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/9128ac15b759492186b34a93d6bd4eb6 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/9128ac15b759492186b34a93d6bd4eb6 2024-12-11T20:34:24,809 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/A of c0a19b15b87adee237235cecb1e4c99f into 9128ac15b759492186b34a93d6bd4eb6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:24,809 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:24,809 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/A, priority=12, startTime=1733949264274; duration=0sec 2024-12-11T20:34:24,809 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:24,809 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:24,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949324838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949324838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949324838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949324838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949324839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:24,915 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/d633b9a20f44491cbc8cb819c99b853c 2024-12-11T20:34:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/94b793d94b3c4a85a654fc5bd896b160 is 50, key is test_row_0/C:col10/1733949263328/Put/seqid=0 2024-12-11T20:34:24,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741889_1065 (size=12151) 2024-12-11T20:34:25,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949325157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949325157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949325158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949325158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949325162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,379 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/94b793d94b3c4a85a654fc5bd896b160 2024-12-11T20:34:25,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/228f67d04821478fab6490ea59836246 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/228f67d04821478fab6490ea59836246 2024-12-11T20:34:25,393 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/228f67d04821478fab6490ea59836246, entries=150, sequenceid=254, filesize=11.9 K 2024-12-11T20:34:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/d633b9a20f44491cbc8cb819c99b853c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d633b9a20f44491cbc8cb819c99b853c 2024-12-11T20:34:25,426 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d633b9a20f44491cbc8cb819c99b853c, entries=150, sequenceid=254, filesize=11.9 K 2024-12-11T20:34:25,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/94b793d94b3c4a85a654fc5bd896b160 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/94b793d94b3c4a85a654fc5bd896b160 2024-12-11T20:34:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T20:34:25,448 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/94b793d94b3c4a85a654fc5bd896b160, entries=150, sequenceid=254, filesize=11.9 K 2024-12-11T20:34:25,449 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for c0a19b15b87adee237235cecb1e4c99f in 1060ms, sequenceid=254, compaction requested=false 2024-12-11T20:34:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:25,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:25,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-11T20:34:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-11T20:34:25,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-11T20:34:25,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1580 sec 2024-12-11T20:34:25,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.1650 sec 2024-12-11T20:34:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:25,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-11T20:34:25,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:25,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:25,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:25,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:25,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:25,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:25,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/300f2d32ade447a2bb5df575b0ed22f2 is 50, key is test_row_0/A:col10/1733949265669/Put/seqid=0 2024-12-11T20:34:25,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949325689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949325689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949325690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949325692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,698 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949325692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741890_1066 (size=14741) 2024-12-11T20:34:25,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949325797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949325806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949325806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949325807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:25,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949325807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:25,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949326007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949326017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949326018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949326018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949326018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/300f2d32ade447a2bb5df575b0ed22f2 2024-12-11T20:34:26,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/4ffec81a4a3e4ea08b7f23791f19404c is 50, key is test_row_0/B:col10/1733949265669/Put/seqid=0 2024-12-11T20:34:26,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741891_1067 (size=12301) 2024-12-11T20:34:26,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949326317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949326327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949326327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949326327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949326328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/4ffec81a4a3e4ea08b7f23791f19404c 2024-12-11T20:34:26,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/af11fbd239944f3688a64f03607b3e38 is 50, key is test_row_0/C:col10/1733949265669/Put/seqid=0 2024-12-11T20:34:26,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741892_1068 (size=12301) 2024-12-11T20:34:26,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/af11fbd239944f3688a64f03607b3e38 2024-12-11T20:34:26,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/300f2d32ade447a2bb5df575b0ed22f2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/300f2d32ade447a2bb5df575b0ed22f2 2024-12-11T20:34:26,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/300f2d32ade447a2bb5df575b0ed22f2, entries=200, sequenceid=275, filesize=14.4 K 2024-12-11T20:34:26,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/4ffec81a4a3e4ea08b7f23791f19404c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/4ffec81a4a3e4ea08b7f23791f19404c 2024-12-11T20:34:26,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/4ffec81a4a3e4ea08b7f23791f19404c, entries=150, sequenceid=275, filesize=12.0 K 2024-12-11T20:34:26,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/af11fbd239944f3688a64f03607b3e38 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/af11fbd239944f3688a64f03607b3e38 2024-12-11T20:34:26,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/af11fbd239944f3688a64f03607b3e38, entries=150, sequenceid=275, filesize=12.0 K 2024-12-11T20:34:26,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for c0a19b15b87adee237235cecb1e4c99f in 916ms, sequenceid=275, compaction requested=true 2024-12-11T20:34:26,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:26,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:26,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:26,586 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:26,586 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:26,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:26,588 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:26,588 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:26,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:26,588 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/B is initiating minor compaction (all files) 2024-12-11T20:34:26,588 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/A is initiating minor compaction (all files) 2024-12-11T20:34:26,588 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/B in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:26,588 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/A in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:26,588 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/9128ac15b759492186b34a93d6bd4eb6, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/228f67d04821478fab6490ea59836246, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/300f2d32ade447a2bb5df575b0ed22f2] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=38.7 K 2024-12-11T20:34:26,588 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b5878fde768e4becaed9ae662ba216ff, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d633b9a20f44491cbc8cb819c99b853c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/4ffec81a4a3e4ea08b7f23791f19404c] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=36.3 K 2024-12-11T20:34:26,589 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9128ac15b759492186b34a93d6bd4eb6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733949262563 2024-12-11T20:34:26,589 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting b5878fde768e4becaed9ae662ba216ff, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733949262563 2024-12-11T20:34:26,589 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 228f67d04821478fab6490ea59836246, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733949263303 2024-12-11T20:34:26,591 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d633b9a20f44491cbc8cb819c99b853c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733949263303 2024-12-11T20:34:26,591 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 300f2d32ade447a2bb5df575b0ed22f2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733949264496 2024-12-11T20:34:26,592 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ffec81a4a3e4ea08b7f23791f19404c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733949264496 2024-12-11T20:34:26,612 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#B#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:26,613 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/c87fcc961186489dbe1507c449fe8787 is 50, key is test_row_0/B:col10/1733949265669/Put/seqid=0 2024-12-11T20:34:26,621 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:26,622 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/10a524e1e0924d34834259e50395a90a is 50, key is test_row_0/A:col10/1733949265669/Put/seqid=0 2024-12-11T20:34:26,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741893_1069 (size=12949) 2024-12-11T20:34:26,680 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/c87fcc961186489dbe1507c449fe8787 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/c87fcc961186489dbe1507c449fe8787 2024-12-11T20:34:26,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741894_1070 (size=12949) 2024-12-11T20:34:26,692 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/B of c0a19b15b87adee237235cecb1e4c99f into c87fcc961186489dbe1507c449fe8787(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:26,693 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:26,693 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/B, priority=13, startTime=1733949266586; duration=0sec 2024-12-11T20:34:26,693 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:26,693 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:26,693 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:26,694 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/10a524e1e0924d34834259e50395a90a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/10a524e1e0924d34834259e50395a90a 2024-12-11T20:34:26,695 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:26,695 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/C is initiating minor compaction (all files) 2024-12-11T20:34:26,696 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/C in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:26,696 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/961357cb33c54703bda2fcf2da0ea372, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/94b793d94b3c4a85a654fc5bd896b160, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/af11fbd239944f3688a64f03607b3e38] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=36.2 K 2024-12-11T20:34:26,697 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 961357cb33c54703bda2fcf2da0ea372, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1733949262563 2024-12-11T20:34:26,697 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b793d94b3c4a85a654fc5bd896b160, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733949263303 2024-12-11T20:34:26,698 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting af11fbd239944f3688a64f03607b3e38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733949264496 2024-12-11T20:34:26,703 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/A of c0a19b15b87adee237235cecb1e4c99f into 10a524e1e0924d34834259e50395a90a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:26,703 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:26,704 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/A, priority=13, startTime=1733949266586; duration=0sec 2024-12-11T20:34:26,704 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:26,704 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:26,713 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#C#compaction#58 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:26,714 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/69cd5f4edb2c4517ad9e985332b38bcd is 50, key is test_row_0/C:col10/1733949265669/Put/seqid=0 2024-12-11T20:34:26,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741895_1071 (size=12915) 2024-12-11T20:34:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:26,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T20:34:26,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:26,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:26,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:26,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:26,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:26,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:26,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/e2ba2b3f2bcf4d9aad566d7bd82ff7b1 is 50, key is test_row_0/A:col10/1733949265689/Put/seqid=0 2024-12-11T20:34:26,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949326857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949326857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949326858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949326860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949326863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741896_1072 (size=12301) 2024-12-11T20:34:26,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949326968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949326967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949326968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949326979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:26,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949326980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:26,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,147 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/69cd5f4edb2c4517ad9e985332b38bcd as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/69cd5f4edb2c4517ad9e985332b38bcd 2024-12-11T20:34:27,155 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/C of c0a19b15b87adee237235cecb1e4c99f into 69cd5f4edb2c4517ad9e985332b38bcd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:27,155 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:27,155 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/C, priority=13, startTime=1733949266588; duration=0sec 2024-12-11T20:34:27,155 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:27,155 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:27,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949327177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949327177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949327177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949327186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949327186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/e2ba2b3f2bcf4d9aad566d7bd82ff7b1 2024-12-11T20:34:27,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/6ce531b6aa03435f890b4fa26d443677 is 50, key is test_row_0/B:col10/1733949265689/Put/seqid=0 2024-12-11T20:34:27,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741897_1073 (size=12301) 2024-12-11T20:34:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-11T20:34:27,448 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:27,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-11T20:34:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T20:34:27,450 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:27,451 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:27,452 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:27,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949327487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949327487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949327487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949327498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949327498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T20:34:27,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-11T20:34:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:27,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:27,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/6ce531b6aa03435f890b4fa26d443677 2024-12-11T20:34:27,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/7e0260a94ce34b58a095a9f20c7bf5df is 50, key is test_row_0/C:col10/1733949265689/Put/seqid=0 2024-12-11T20:34:27,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741898_1074 (size=12301) 2024-12-11T20:34:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T20:34:27,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-11T20:34:27,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:27,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:27,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:27,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:27,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:27,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-11T20:34:27,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:27,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:27,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:27,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:27,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:27,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949327997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949327997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949327997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:27,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:27,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:28,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:28,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:28,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:28,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949328007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:28,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:28,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:28,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949328017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:28,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T20:34:28,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-11T20:34:28,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:28,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:28,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:28,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:28,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:28,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:28,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/7e0260a94ce34b58a095a9f20c7bf5df 2024-12-11T20:34:28,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/e2ba2b3f2bcf4d9aad566d7bd82ff7b1 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/e2ba2b3f2bcf4d9aad566d7bd82ff7b1 2024-12-11T20:34:28,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/e2ba2b3f2bcf4d9aad566d7bd82ff7b1, entries=150, sequenceid=292, filesize=12.0 K 2024-12-11T20:34:28,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/6ce531b6aa03435f890b4fa26d443677 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6ce531b6aa03435f890b4fa26d443677 2024-12-11T20:34:28,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6ce531b6aa03435f890b4fa26d443677, entries=150, sequenceid=292, filesize=12.0 K 2024-12-11T20:34:28,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/7e0260a94ce34b58a095a9f20c7bf5df as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/7e0260a94ce34b58a095a9f20c7bf5df 2024-12-11T20:34:28,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/7e0260a94ce34b58a095a9f20c7bf5df, entries=150, sequenceid=292, filesize=12.0 K 2024-12-11T20:34:28,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c0a19b15b87adee237235cecb1e4c99f in 1370ms, sequenceid=292, compaction requested=false 2024-12-11T20:34:28,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:28,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-11T20:34:28,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:28,242 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T20:34:28,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:28,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:28,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:28,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:28,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:28,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:28,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/9418e44d81394567b872f4ba6fe7eaf2 is 50, key is test_row_0/A:col10/1733949266858/Put/seqid=0 2024-12-11T20:34:28,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741899_1075 (size=12301) 2024-12-11T20:34:28,308 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/9418e44d81394567b872f4ba6fe7eaf2 2024-12-11T20:34:28,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/de25570c454849d49a3262f0b50c0851 is 50, key is test_row_0/B:col10/1733949266858/Put/seqid=0 2024-12-11T20:34:28,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741900_1076 (size=12301) 2024-12-11T20:34:28,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T20:34:28,731 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/de25570c454849d49a3262f0b50c0851 2024-12-11T20:34:28,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/4e7df3410361438aa2b7fddb89d3d4ab is 50, key is test_row_0/C:col10/1733949266858/Put/seqid=0 2024-12-11T20:34:28,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741901_1077 (size=12301) 2024-12-11T20:34:28,753 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/4e7df3410361438aa2b7fddb89d3d4ab 2024-12-11T20:34:28,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/9418e44d81394567b872f4ba6fe7eaf2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/9418e44d81394567b872f4ba6fe7eaf2 2024-12-11T20:34:28,773 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/9418e44d81394567b872f4ba6fe7eaf2, entries=150, sequenceid=310, filesize=12.0 K 2024-12-11T20:34:28,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/de25570c454849d49a3262f0b50c0851 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/de25570c454849d49a3262f0b50c0851 2024-12-11T20:34:28,781 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/de25570c454849d49a3262f0b50c0851, entries=150, sequenceid=310, filesize=12.0 K 2024-12-11T20:34:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/4e7df3410361438aa2b7fddb89d3d4ab as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/4e7df3410361438aa2b7fddb89d3d4ab 2024-12-11T20:34:28,790 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/4e7df3410361438aa2b7fddb89d3d4ab, entries=150, sequenceid=310, filesize=12.0 K 2024-12-11T20:34:28,791 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for c0a19b15b87adee237235cecb1e4c99f in 548ms, sequenceid=310, compaction requested=true 2024-12-11T20:34:28,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:28,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:28,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-11T20:34:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-11T20:34:28,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-11T20:34:28,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3400 sec 2024-12-11T20:34:28,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.3480 sec 2024-12-11T20:34:29,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:34:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:29,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:29,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:29,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:29,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:29,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:29,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:29,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/5c47876c87bb4082a9231c6ada150d95 is 50, key is test_row_0/A:col10/1733949269022/Put/seqid=0 2024-12-11T20:34:29,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741902_1078 (size=17181) 2024-12-11T20:34:29,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/5c47876c87bb4082a9231c6ada150d95 2024-12-11T20:34:29,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/07c6932d51b34a6ea4283afed6e0653b is 50, key is test_row_0/B:col10/1733949269022/Put/seqid=0 2024-12-11T20:34:29,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949329081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949329087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949329087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,101 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949329089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741903_1079 (size=12301) 2024-12-11T20:34:29,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949269078, max timestamp=1733949269087Num uniques 859; 2024-12-11T20:34:29,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=7688, suffix heap size=117064,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=168 2024-12-11T20:34:29,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/07c6932d51b34a6ea4283afed6e0653b 2024-12-11T20:34:29,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949269084, max timestamp=1733949269087Num uniques 859; 2024-12-11T20:34:29,121 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:29,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949269084, max timestamp=1733949269087Num uniques 859; 2024-12-11T20:34:29,122 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:29,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/1d8d495ad414465c8b11b9a527a26788 is 50, key is test_row_0/C:col10/1733949269022/Put/seqid=0 2024-12-11T20:34:29,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741904_1080 (size=12301) 2024-12-11T20:34:29,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949329200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949329207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949329207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949329208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949329211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949329316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949329417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,426 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949329418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949329419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949329419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949329527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,559 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/1d8d495ad414465c8b11b9a527a26788 2024-12-11T20:34:29,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/5c47876c87bb4082a9231c6ada150d95 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/5c47876c87bb4082a9231c6ada150d95 2024-12-11T20:34:29,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/5c47876c87bb4082a9231c6ada150d95, entries=250, sequenceid=322, filesize=16.8 K 2024-12-11T20:34:29,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/07c6932d51b34a6ea4283afed6e0653b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/07c6932d51b34a6ea4283afed6e0653b 2024-12-11T20:34:29,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/07c6932d51b34a6ea4283afed6e0653b, entries=150, sequenceid=322, filesize=12.0 K 2024-12-11T20:34:29,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/1d8d495ad414465c8b11b9a527a26788 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1d8d495ad414465c8b11b9a527a26788 2024-12-11T20:34:29,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1d8d495ad414465c8b11b9a527a26788, entries=150, sequenceid=322, filesize=12.0 K 2024-12-11T20:34:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-11T20:34:29,598 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:29,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=121.17 KB/124074 for c0a19b15b87adee237235cecb1e4c99f in 576ms, sequenceid=322, compaction requested=true 2024-12-11T20:34:29,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:29,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:29,598 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:29,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:29,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:29,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:29,598 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:29,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:29,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:29,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-11T20:34:29,602 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T20:34:29,603 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:29,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:29,604 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:29,604 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/B is initiating minor compaction (all files) 2024-12-11T20:34:29,604 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/B in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:29,604 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/c87fcc961186489dbe1507c449fe8787, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6ce531b6aa03435f890b4fa26d443677, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/de25570c454849d49a3262f0b50c0851, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/07c6932d51b34a6ea4283afed6e0653b] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=48.7 K 2024-12-11T20:34:29,604 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54732 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:29,605 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/A is initiating minor compaction (all files) 2024-12-11T20:34:29,605 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/A in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:29,605 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/10a524e1e0924d34834259e50395a90a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/e2ba2b3f2bcf4d9aad566d7bd82ff7b1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/9418e44d81394567b872f4ba6fe7eaf2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/5c47876c87bb4082a9231c6ada150d95] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=53.4 K 2024-12-11T20:34:29,606 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c87fcc961186489dbe1507c449fe8787, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733949264496 2024-12-11T20:34:29,606 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10a524e1e0924d34834259e50395a90a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733949264496 2024-12-11T20:34:29,606 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce531b6aa03435f890b4fa26d443677, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733949265686 2024-12-11T20:34:29,606 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2ba2b3f2bcf4d9aad566d7bd82ff7b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733949265686 2024-12-11T20:34:29,607 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting de25570c454849d49a3262f0b50c0851, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733949266856 2024-12-11T20:34:29,607 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9418e44d81394567b872f4ba6fe7eaf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733949266856 2024-12-11T20:34:29,607 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 07c6932d51b34a6ea4283afed6e0653b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733949269014 2024-12-11T20:34:29,607 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c47876c87bb4082a9231c6ada150d95, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733949269014 2024-12-11T20:34:29,625 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#A#compaction#68 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:29,626 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/356a54362f6f4f208e3f7d5a1d59e994 is 50, key is test_row_0/A:col10/1733949269022/Put/seqid=0 2024-12-11T20:34:29,637 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:29,638 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/6c1aad6406b444268ac657f13904cb65 is 50, key is test_row_0/B:col10/1733949269022/Put/seqid=0 2024-12-11T20:34:29,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741905_1081 (size=13085) 2024-12-11T20:34:29,657 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/356a54362f6f4f208e3f7d5a1d59e994 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/356a54362f6f4f208e3f7d5a1d59e994 2024-12-11T20:34:29,663 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/A of c0a19b15b87adee237235cecb1e4c99f into 356a54362f6f4f208e3f7d5a1d59e994(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:29,663 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:29,663 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/A, priority=12, startTime=1733949269598; duration=0sec 2024-12-11T20:34:29,664 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:29,664 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:29,664 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:29,665 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:29,665 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/C is initiating minor compaction (all files) 2024-12-11T20:34:29,665 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/C in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:29,665 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/69cd5f4edb2c4517ad9e985332b38bcd, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/7e0260a94ce34b58a095a9f20c7bf5df, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/4e7df3410361438aa2b7fddb89d3d4ab, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1d8d495ad414465c8b11b9a527a26788] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=48.7 K 2024-12-11T20:34:29,667 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69cd5f4edb2c4517ad9e985332b38bcd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1733949264496 2024-12-11T20:34:29,668 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e0260a94ce34b58a095a9f20c7bf5df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1733949265686 2024-12-11T20:34:29,668 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e7df3410361438aa2b7fddb89d3d4ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733949266856 2024-12-11T20:34:29,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741906_1082 (size=13085) 2024-12-11T20:34:29,669 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d8d495ad414465c8b11b9a527a26788, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733949269014 2024-12-11T20:34:29,689 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#C#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:29,690 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/212aa9c0243f485b97201608e743bf46 is 50, key is test_row_0/C:col10/1733949269022/Put/seqid=0 2024-12-11T20:34:29,696 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/6c1aad6406b444268ac657f13904cb65 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6c1aad6406b444268ac657f13904cb65 2024-12-11T20:34:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T20:34:29,711 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/B of c0a19b15b87adee237235cecb1e4c99f into 6c1aad6406b444268ac657f13904cb65(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:29,711 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:29,711 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/B, priority=12, startTime=1733949269598; duration=0sec 2024-12-11T20:34:29,711 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:29,711 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:29,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741907_1083 (size=13051) 2024-12-11T20:34:29,729 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/212aa9c0243f485b97201608e743bf46 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/212aa9c0243f485b97201608e743bf46 2024-12-11T20:34:29,735 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/C of c0a19b15b87adee237235cecb1e4c99f into 212aa9c0243f485b97201608e743bf46(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:29,735 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:29,735 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/C, priority=12, startTime=1733949269598; duration=0sec 2024-12-11T20:34:29,736 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:29,736 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:29,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:29,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=127.88 KB heapSize=365.98 KB 2024-12-11T20:34:29,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:29,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:29,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:29,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:29,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:29,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:29,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/7d7debc714724281bff9ae19f48b2b5e is 50, key is test_row_0/A:col10/1733949269198/Put/seqid=0 2024-12-11T20:34:29,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T20:34:29,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:29,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:29,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:29,759 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:29,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:29,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949329757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949329760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949329760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949329765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741908_1084 (size=17181) 2024-12-11T20:34:29,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949329836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949329877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949329878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949329878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949329878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:29,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:29,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:29,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T20:34:29,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:29,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:29,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:29,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:29,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:29,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:29,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T20:34:30,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T20:34:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949330087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949330087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949330097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949330097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,098 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45.13 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/7d7debc714724281bff9ae19f48b2b5e 2024-12-11T20:34:30,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/c00a100c016f426f8b4ff47a1cd6ea45 is 50, key is test_row_0/B:col10/1733949269198/Put/seqid=0 2024-12-11T20:34:30,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741909_1085 (size=12301) 2024-12-11T20:34:30,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T20:34:30,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T20:34:30,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949330347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T20:34:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949330397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949330397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949330408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,412 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949330408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T20:34:30,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:30,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/c00a100c016f426f8b4ff47a1cd6ea45 2024-12-11T20:34:30,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/a0ae5c74fb4c4345908c0e2680ed2136 is 50, key is test_row_0/C:col10/1733949269198/Put/seqid=0 2024-12-11T20:34:30,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741910_1086 (size=12301) 2024-12-11T20:34:30,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T20:34:30,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:30,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T20:34:30,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T20:34:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:30,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:30,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:30,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949330908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949330908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949330917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:30,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:30,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949330927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:30,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:31,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T20:34:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:31,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:31,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:31,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:31,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/a0ae5c74fb4c4345908c0e2680ed2136 2024-12-11T20:34:31,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/7d7debc714724281bff9ae19f48b2b5e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7d7debc714724281bff9ae19f48b2b5e 2024-12-11T20:34:31,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7d7debc714724281bff9ae19f48b2b5e, entries=250, sequenceid=362, filesize=16.8 K 2024-12-11T20:34:31,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/c00a100c016f426f8b4ff47a1cd6ea45 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/c00a100c016f426f8b4ff47a1cd6ea45 2024-12-11T20:34:31,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/c00a100c016f426f8b4ff47a1cd6ea45, entries=150, sequenceid=362, filesize=12.0 K 2024-12-11T20:34:31,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/a0ae5c74fb4c4345908c0e2680ed2136 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/a0ae5c74fb4c4345908c0e2680ed2136 2024-12-11T20:34:31,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/a0ae5c74fb4c4345908c0e2680ed2136, entries=150, sequenceid=362, filesize=12.0 K 2024-12-11T20:34:31,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.58 KB/137814, heapSize ~385.85 KB/395112, currentSize=46.96 KB/48090 for c0a19b15b87adee237235cecb1e4c99f in 1363ms, sequenceid=362, compaction requested=false 2024-12-11T20:34:31,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:31,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-11T20:34:31,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:31,157 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:34:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:31,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:31,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:31,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:31,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/034b478569cf4dc0bea0e9980646ac0d is 50, key is test_row_0/A:col10/1733949269757/Put/seqid=0 2024-12-11T20:34:31,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741911_1087 (size=12301) 2024-12-11T20:34:31,225 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/034b478569cf4dc0bea0e9980646ac0d 2024-12-11T20:34:31,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/7ffc0dd14adc4ea5b2a5fd4222f35759 is 50, key is test_row_0/B:col10/1733949269757/Put/seqid=0 2024-12-11T20:34:31,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741912_1088 (size=12301) 2024-12-11T20:34:31,257 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/7ffc0dd14adc4ea5b2a5fd4222f35759 2024-12-11T20:34:31,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/411b18d9fca44904a3c2057256297e06 is 50, key is test_row_0/C:col10/1733949269757/Put/seqid=0 2024-12-11T20:34:31,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741913_1089 (size=12301) 2024-12-11T20:34:31,317 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/411b18d9fca44904a3c2057256297e06 2024-12-11T20:34:31,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/034b478569cf4dc0bea0e9980646ac0d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/034b478569cf4dc0bea0e9980646ac0d 2024-12-11T20:34:31,335 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/034b478569cf4dc0bea0e9980646ac0d, entries=150, sequenceid=372, filesize=12.0 K 2024-12-11T20:34:31,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/7ffc0dd14adc4ea5b2a5fd4222f35759 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ffc0dd14adc4ea5b2a5fd4222f35759 2024-12-11T20:34:31,347 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ffc0dd14adc4ea5b2a5fd4222f35759, entries=150, sequenceid=372, filesize=12.0 K 2024-12-11T20:34:31,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/411b18d9fca44904a3c2057256297e06 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/411b18d9fca44904a3c2057256297e06 2024-12-11T20:34:31,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:31,364 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/411b18d9fca44904a3c2057256297e06, entries=150, sequenceid=372, filesize=12.0 K 2024-12-11T20:34:31,366 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=6.71 KB/6870 for c0a19b15b87adee237235cecb1e4c99f in 209ms, sequenceid=372, compaction requested=true 2024-12-11T20:34:31,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:31,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:31,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-11T20:34:31,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-11T20:34:31,373 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-11T20:34:31,373 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7650 sec 2024-12-11T20:34:31,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.7770 sec 2024-12-11T20:34:31,403 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:34:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:31,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:31,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:31,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/80ed7d6634bb417896ed91637f3b9660 is 50, key is test_row_0/A:col10/1733949271395/Put/seqid=0 2024-12-11T20:34:31,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741914_1090 (size=14741) 2024-12-11T20:34:31,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949271484, max timestamp=1733949271518Num uniques 859; 2024-12-11T20:34:31,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949271484, max timestamp=1733949271518Num uniques 859; 2024-12-11T20:34:31,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949271484, max timestamp=1733949271518Num uniques 859; 2024-12-11T20:34:31,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:31,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:31,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:31,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:31,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949331596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:31,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949331706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:31,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-11T20:34:31,747 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:31,748 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-11T20:34:31,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T20:34:31,750 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:31,755 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:31,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:31,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/80ed7d6634bb417896ed91637f3b9660 2024-12-11T20:34:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T20:34:31,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/f87c9a06f1fd47d3b17f6d93ecd14f8c is 50, key is test_row_0/B:col10/1733949271395/Put/seqid=0 2024-12-11T20:34:31,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741915_1091 (size=12301) 2024-12-11T20:34:31,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-11T20:34:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:31,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:31,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:31,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:31,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:31,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949331917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:31,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:31,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949331917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:31,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949331917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:31,922 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in c0a19b15b87adee237235cecb1e4c99f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:31,923 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in c0a19b15b87adee237235cecb1e4c99f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:31,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:31,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949331937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:31,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:31,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949331937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:31,943 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0a19b15b87adee237235cecb1e4c99f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,944 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:31,944 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0a19b15b87adee237235cecb1e4c99f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:31,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:32,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-11T20:34:32,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:32,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:32,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:32,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:32,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T20:34:32,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-11T20:34:32,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:32,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:32,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:32,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:32,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:32,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:32,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949332237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:32,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:32,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:32,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/f87c9a06f1fd47d3b17f6d93ecd14f8c 2024-12-11T20:34:32,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/8116a2d8432f4771b690865313fbb98b is 50, key is test_row_0/C:col10/1733949271395/Put/seqid=0 2024-12-11T20:34:32,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741916_1092 (size=12301) 2024-12-11T20:34:32,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/8116a2d8432f4771b690865313fbb98b 2024-12-11T20:34:32,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/80ed7d6634bb417896ed91637f3b9660 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/80ed7d6634bb417896ed91637f3b9660 2024-12-11T20:34:32,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/80ed7d6634bb417896ed91637f3b9660, entries=200, sequenceid=382, filesize=14.4 K 2024-12-11T20:34:32,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/f87c9a06f1fd47d3b17f6d93ecd14f8c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f87c9a06f1fd47d3b17f6d93ecd14f8c 2024-12-11T20:34:32,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f87c9a06f1fd47d3b17f6d93ecd14f8c, entries=150, sequenceid=382, filesize=12.0 K 2024-12-11T20:34:32,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/8116a2d8432f4771b690865313fbb98b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/8116a2d8432f4771b690865313fbb98b 2024-12-11T20:34:32,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/8116a2d8432f4771b690865313fbb98b, entries=150, sequenceid=382, filesize=12.0 K 2024-12-11T20:34:32,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c0a19b15b87adee237235cecb1e4c99f in 927ms, sequenceid=382, compaction requested=true 2024-12-11T20:34:32,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:32,331 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:32,331 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:32,332 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57308 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:32,332 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:32,332 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/A is initiating minor compaction (all files) 2024-12-11T20:34:32,332 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/B is initiating minor compaction (all files) 2024-12-11T20:34:32,333 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/A in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:32,333 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/B in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:32,333 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/356a54362f6f4f208e3f7d5a1d59e994, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7d7debc714724281bff9ae19f48b2b5e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/034b478569cf4dc0bea0e9980646ac0d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/80ed7d6634bb417896ed91637f3b9660] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=56.0 K 2024-12-11T20:34:32,333 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6c1aad6406b444268ac657f13904cb65, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/c00a100c016f426f8b4ff47a1cd6ea45, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ffc0dd14adc4ea5b2a5fd4222f35759, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f87c9a06f1fd47d3b17f6d93ecd14f8c] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=48.8 K 2024-12-11T20:34:32,333 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 356a54362f6f4f208e3f7d5a1d59e994, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733949269014 2024-12-11T20:34:32,333 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c1aad6406b444268ac657f13904cb65, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733949269014 2024-12-11T20:34:32,334 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c00a100c016f426f8b4ff47a1cd6ea45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733949269198 2024-12-11T20:34:32,334 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d7debc714724281bff9ae19f48b2b5e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733949269154 2024-12-11T20:34:32,334 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ffc0dd14adc4ea5b2a5fd4222f35759, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733949269749 2024-12-11T20:34:32,334 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 034b478569cf4dc0bea0e9980646ac0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733949269749 2024-12-11T20:34:32,334 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f87c9a06f1fd47d3b17f6d93ecd14f8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733949271390 2024-12-11T20:34:32,334 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80ed7d6634bb417896ed91637f3b9660, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733949271383 2024-12-11T20:34:32,345 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#A#compaction#80 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:32,346 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/3e75c2d4504f410aa6b227d5e4df5161 is 50, key is test_row_0/A:col10/1733949271395/Put/seqid=0 2024-12-11T20:34:32,349 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#B#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:32,350 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/d2118cb60e9b4894995ae16db664cbbc is 50, key is test_row_0/B:col10/1733949271395/Put/seqid=0 2024-12-11T20:34:32,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741917_1093 (size=13221) 2024-12-11T20:34:32,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741918_1094 (size=13221) 2024-12-11T20:34:32,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-11T20:34:32,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:32,372 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-11T20:34:32,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:32,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T20:34:32,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/1cd47106850d46a0b4f9680d4688ef8f is 50, key is test_row_0/A:col10/1733949271595/Put/seqid=0 2024-12-11T20:34:32,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741919_1095 (size=12301) 2024-12-11T20:34:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:32,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:32,771 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/3e75c2d4504f410aa6b227d5e4df5161 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/3e75c2d4504f410aa6b227d5e4df5161 2024-12-11T20:34:32,780 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/A of c0a19b15b87adee237235cecb1e4c99f into 3e75c2d4504f410aa6b227d5e4df5161(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:32,781 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:32,781 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/A, priority=12, startTime=1733949272331; duration=0sec 2024-12-11T20:34:32,781 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:32,781 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:32,781 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:32,783 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/d2118cb60e9b4894995ae16db664cbbc as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d2118cb60e9b4894995ae16db664cbbc 2024-12-11T20:34:32,785 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/1cd47106850d46a0b4f9680d4688ef8f 2024-12-11T20:34:32,786 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:32,786 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/C is initiating minor compaction (all files) 2024-12-11T20:34:32,786 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/C in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:32,786 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/212aa9c0243f485b97201608e743bf46, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/a0ae5c74fb4c4345908c0e2680ed2136, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/411b18d9fca44904a3c2057256297e06, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/8116a2d8432f4771b690865313fbb98b] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=48.8 K 2024-12-11T20:34:32,787 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 212aa9c0243f485b97201608e743bf46, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733949269014 2024-12-11T20:34:32,788 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0ae5c74fb4c4345908c0e2680ed2136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1733949269198 2024-12-11T20:34:32,789 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 411b18d9fca44904a3c2057256297e06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1733949269749 2024-12-11T20:34:32,790 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8116a2d8432f4771b690865313fbb98b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733949271390 2024-12-11T20:34:32,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:32,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949332789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:32,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:32,798 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/B of c0a19b15b87adee237235cecb1e4c99f into d2118cb60e9b4894995ae16db664cbbc(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:32,798 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:32,798 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/B, priority=12, startTime=1733949272331; duration=0sec 2024-12-11T20:34:32,799 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:32,799 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:32,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/fbb61589f4b24ed19f991387c68b0d48 is 50, key is test_row_0/B:col10/1733949271595/Put/seqid=0 2024-12-11T20:34:32,824 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#C#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:32,824 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/d1444fdbb8634518b373945ed7acda4e is 50, key is test_row_0/C:col10/1733949271395/Put/seqid=0 2024-12-11T20:34:32,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741920_1096 (size=12301) 2024-12-11T20:34:32,877 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/fbb61589f4b24ed19f991387c68b0d48 2024-12-11T20:34:32,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T20:34:32,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949332897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:32,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:32,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741921_1097 (size=13187) 2024-12-11T20:34:32,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/78dd05ea840943888b39ae23d6275098 is 50, key is test_row_0/C:col10/1733949271595/Put/seqid=0 2024-12-11T20:34:32,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741922_1098 (size=12301) 2024-12-11T20:34:33,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949333106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:33,314 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/d1444fdbb8634518b373945ed7acda4e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d1444fdbb8634518b373945ed7acda4e 2024-12-11T20:34:33,328 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/C of c0a19b15b87adee237235cecb1e4c99f into d1444fdbb8634518b373945ed7acda4e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:33,328 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:33,328 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/C, priority=12, startTime=1733949272331; duration=0sec 2024-12-11T20:34:33,329 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:33,329 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:33,359 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/78dd05ea840943888b39ae23d6275098 2024-12-11T20:34:33,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/1cd47106850d46a0b4f9680d4688ef8f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1cd47106850d46a0b4f9680d4688ef8f 2024-12-11T20:34:33,382 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1cd47106850d46a0b4f9680d4688ef8f, entries=150, sequenceid=419, filesize=12.0 K 2024-12-11T20:34:33,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/fbb61589f4b24ed19f991387c68b0d48 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/fbb61589f4b24ed19f991387c68b0d48 2024-12-11T20:34:33,398 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/fbb61589f4b24ed19f991387c68b0d48, entries=150, sequenceid=419, filesize=12.0 K 2024-12-11T20:34:33,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/78dd05ea840943888b39ae23d6275098 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/78dd05ea840943888b39ae23d6275098 2024-12-11T20:34:33,410 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/78dd05ea840943888b39ae23d6275098, entries=150, sequenceid=419, filesize=12.0 K 2024-12-11T20:34:33,412 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c0a19b15b87adee237235cecb1e4c99f in 1040ms, sequenceid=419, compaction requested=false 2024-12-11T20:34:33,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:33,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:33,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-11T20:34:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-11T20:34:33,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-11T20:34:33,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6600 sec 2024-12-11T20:34:33,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T20:34:33,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:33,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:33,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:33,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.6720 sec 2024-12-11T20:34:33,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/14ba6017647a4a13a9d82cc28897c5de is 50, key is test_row_0/A:col10/1733949272776/Put/seqid=0 2024-12-11T20:34:33,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741923_1099 (size=12301) 2024-12-11T20:34:33,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/14ba6017647a4a13a9d82cc28897c5de 2024-12-11T20:34:33,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/613744e9b11a41d987af770dc73b61c8 is 50, key is test_row_0/B:col10/1733949272776/Put/seqid=0 2024-12-11T20:34:33,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741924_1100 (size=12301) 2024-12-11T20:34:33,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/613744e9b11a41d987af770dc73b61c8 2024-12-11T20:34:33,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/c965a82bec304bc18fe566e3586a2102 is 50, key is test_row_0/C:col10/1733949272776/Put/seqid=0 2024-12-11T20:34:33,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733949273527, max timestamp=1733949273550Num uniques 859; 2024-12-11T20:34:33,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733949273527, max timestamp=1733949273550Num uniques 859; 2024-12-11T20:34:33,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T20:34:33,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T20:34:33,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733949273527, max timestamp=1733949273550Num uniques 859; 2024-12-11T20:34:33,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T20:34:33,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741925_1101 (size=12301) 2024-12-11T20:34:33,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/c965a82bec304bc18fe566e3586a2102 2024-12-11T20:34:33,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/14ba6017647a4a13a9d82cc28897c5de as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/14ba6017647a4a13a9d82cc28897c5de 2024-12-11T20:34:33,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/14ba6017647a4a13a9d82cc28897c5de, entries=150, sequenceid=433, filesize=12.0 K 2024-12-11T20:34:33,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/613744e9b11a41d987af770dc73b61c8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/613744e9b11a41d987af770dc73b61c8 2024-12-11T20:34:33,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/613744e9b11a41d987af770dc73b61c8, entries=150, sequenceid=433, filesize=12.0 K 2024-12-11T20:34:33,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/c965a82bec304bc18fe566e3586a2102 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c965a82bec304bc18fe566e3586a2102 2024-12-11T20:34:33,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c965a82bec304bc18fe566e3586a2102, entries=150, sequenceid=433, filesize=12.0 K 2024-12-11T20:34:33,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=93.93 KB/96180 for c0a19b15b87adee237235cecb1e4c99f in 199ms, sequenceid=433, compaction requested=true 2024-12-11T20:34:33,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:33,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:33,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:33,620 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:33,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:33,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:33,620 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:33,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:33,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:33,621 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:33,621 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:33,621 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/A is initiating minor compaction (all files) 2024-12-11T20:34:33,621 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/B is initiating minor compaction (all files) 2024-12-11T20:34:33,621 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/B in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:33,621 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/A in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:33,622 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d2118cb60e9b4894995ae16db664cbbc, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/fbb61589f4b24ed19f991387c68b0d48, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/613744e9b11a41d987af770dc73b61c8] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=36.9 K 2024-12-11T20:34:33,622 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/3e75c2d4504f410aa6b227d5e4df5161, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1cd47106850d46a0b4f9680d4688ef8f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/14ba6017647a4a13a9d82cc28897c5de] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=36.9 K 2024-12-11T20:34:33,622 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d2118cb60e9b4894995ae16db664cbbc, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733949271390 2024-12-11T20:34:33,622 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e75c2d4504f410aa6b227d5e4df5161, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733949271390 2024-12-11T20:34:33,622 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb61589f4b24ed19f991387c68b0d48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733949271587 2024-12-11T20:34:33,622 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cd47106850d46a0b4f9680d4688ef8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733949271587 2024-12-11T20:34:33,623 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 613744e9b11a41d987af770dc73b61c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733949272776 2024-12-11T20:34:33,623 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14ba6017647a4a13a9d82cc28897c5de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733949272776 2024-12-11T20:34:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:33,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=100.63 KB heapSize=280.69 KB 2024-12-11T20:34:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:33,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:33,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:33,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:33,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:33,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:33,634 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#A#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:33,635 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/62b46ac1332043a89447249c916d792f is 50, key is test_row_0/A:col10/1733949272776/Put/seqid=0 2024-12-11T20:34:33,638 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:33,639 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/b1438514cb324627ba34a954e3b553be is 50, key is test_row_0/B:col10/1733949272776/Put/seqid=0 2024-12-11T20:34:33,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/54a7521be4dc44cdb1177e210db28718 is 50, key is test_row_0/A:col10/1733949273610/Put/seqid=0 2024-12-11T20:34:33,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741927_1103 (size=13323) 2024-12-11T20:34:33,716 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/b1438514cb324627ba34a954e3b553be as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b1438514cb324627ba34a954e3b553be 2024-12-11T20:34:33,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949333715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:33,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741926_1102 (size=13323) 2024-12-11T20:34:33,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741928_1104 (size=14741) 2024-12-11T20:34:33,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/54a7521be4dc44cdb1177e210db28718 2024-12-11T20:34:33,732 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/B of c0a19b15b87adee237235cecb1e4c99f into b1438514cb324627ba34a954e3b553be(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:33,732 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:33,732 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/B, priority=13, startTime=1733949273620; duration=0sec 2024-12-11T20:34:33,733 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:33,733 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:33,733 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:33,736 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:33,737 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/C is initiating minor compaction (all files) 2024-12-11T20:34:33,738 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/C in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:33,739 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d1444fdbb8634518b373945ed7acda4e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/78dd05ea840943888b39ae23d6275098, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c965a82bec304bc18fe566e3586a2102] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=36.9 K 2024-12-11T20:34:33,740 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d1444fdbb8634518b373945ed7acda4e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733949271390 2024-12-11T20:34:33,740 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 78dd05ea840943888b39ae23d6275098, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733949271587 2024-12-11T20:34:33,743 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c965a82bec304bc18fe566e3586a2102, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733949272776 2024-12-11T20:34:33,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/8156ff0571734a7f8c5ec158de533880 is 50, key is test_row_0/B:col10/1733949273610/Put/seqid=0 2024-12-11T20:34:33,788 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:33,790 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/52bfa0d23a4b4ae5b7f90f23e47ee0ac is 50, key is test_row_0/C:col10/1733949272776/Put/seqid=0 2024-12-11T20:34:33,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741929_1105 (size=12301) 2024-12-11T20:34:33,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/8156ff0571734a7f8c5ec158de533880 2024-12-11T20:34:33,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741930_1106 (size=13289) 2024-12-11T20:34:33,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949333827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:33,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/797428bde4724bd8bd747196ec37ed1d is 50, key is test_row_0/C:col10/1733949273610/Put/seqid=0 2024-12-11T20:34:33,859 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/52bfa0d23a4b4ae5b7f90f23e47ee0ac as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/52bfa0d23a4b4ae5b7f90f23e47ee0ac 2024-12-11T20:34:33,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741931_1107 (size=12301) 2024-12-11T20:34:33,888 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/C of c0a19b15b87adee237235cecb1e4c99f into 52bfa0d23a4b4ae5b7f90f23e47ee0ac(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:33,888 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:33,888 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/C, priority=13, startTime=1733949273620; duration=0sec 2024-12-11T20:34:33,888 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:33,888 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-11T20:34:33,898 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:33,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-11T20:34:33,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T20:34:33,900 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:33,901 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:33,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:33,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57016 deadline: 1733949333948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:33,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56976 deadline: 1733949333948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:33,951 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in c0a19b15b87adee237235cecb1e4c99f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:33,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:33,951 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in c0a19b15b87adee237235cecb1e4c99f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:33,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:33,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56992 deadline: 1733949333957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:33,963 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0a19b15b87adee237235cecb1e4c99f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:33,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,964 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:33,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56994 deadline: 1733949333967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:33,969 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0a19b15b87adee237235cecb1e4c99f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T20:34:34,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949334038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:34,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:34,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:34,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:34,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-11T20:34:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,132 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/62b46ac1332043a89447249c916d792f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/62b46ac1332043a89447249c916d792f 2024-12-11T20:34:34,152 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/A of c0a19b15b87adee237235cecb1e4c99f into 62b46ac1332043a89447249c916d792f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:34,152 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:34,152 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/A, priority=13, startTime=1733949273620; duration=0sec 2024-12-11T20:34:34,152 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:34,152 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:34,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-11T20:34:34,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:34,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T20:34:34,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/797428bde4724bd8bd747196ec37ed1d 2024-12-11T20:34:34,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/54a7521be4dc44cdb1177e210db28718 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/54a7521be4dc44cdb1177e210db28718 2024-12-11T20:34:34,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/54a7521be4dc44cdb1177e210db28718, entries=200, sequenceid=464, filesize=14.4 K 2024-12-11T20:34:34,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/8156ff0571734a7f8c5ec158de533880 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8156ff0571734a7f8c5ec158de533880 2024-12-11T20:34:34,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8156ff0571734a7f8c5ec158de533880, entries=150, sequenceid=464, filesize=12.0 K 2024-12-11T20:34:34,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/797428bde4724bd8bd747196ec37ed1d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/797428bde4724bd8bd747196ec37ed1d 2024-12-11T20:34:34,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/797428bde4724bd8bd747196ec37ed1d, entries=150, sequenceid=464, filesize=12.0 K 2024-12-11T20:34:34,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~280.64 KB/287376, currentSize=80.51 KB/82440 for c0a19b15b87adee237235cecb1e4c99f in 711ms, sequenceid=464, compaction requested=false 2024-12-11T20:34:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:34,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:34:34,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:34,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:34,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:34,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:34,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:34,354 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:34,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-11T20:34:34,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:34,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/4ed335ffecb3435da6ada2b4cafcbcec is 50, key is test_row_0/A:col10/1733949273640/Put/seqid=0 2024-12-11T20:34:34,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:34,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741932_1108 (size=12301) 2024-12-11T20:34:34,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949334416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:34,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:34,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/4ed335ffecb3435da6ada2b4cafcbcec 2024-12-11T20:34:34,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/bddba88249154d43a38dbca1c3bfe1f3 is 50, key is test_row_0/B:col10/1733949273640/Put/seqid=0 2024-12-11T20:34:34,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741933_1109 (size=12301) 2024-12-11T20:34:34,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/bddba88249154d43a38dbca1c3bfe1f3 2024-12-11T20:34:34,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/1554a13c60d54d71b8d5f129f99be20f is 50, key is test_row_0/C:col10/1733949273640/Put/seqid=0 2024-12-11T20:34:34,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741934_1110 (size=12301) 2024-12-11T20:34:34,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/1554a13c60d54d71b8d5f129f99be20f 2024-12-11T20:34:34,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-11T20:34:34,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:34,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T20:34:34,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/4ed335ffecb3435da6ada2b4cafcbcec as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/4ed335ffecb3435da6ada2b4cafcbcec 2024-12-11T20:34:34,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949334527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:34,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:34,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/4ed335ffecb3435da6ada2b4cafcbcec, entries=150, sequenceid=483, filesize=12.0 K 2024-12-11T20:34:34,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/bddba88249154d43a38dbca1c3bfe1f3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bddba88249154d43a38dbca1c3bfe1f3 2024-12-11T20:34:34,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bddba88249154d43a38dbca1c3bfe1f3, entries=150, sequenceid=483, filesize=12.0 K 2024-12-11T20:34:34,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/1554a13c60d54d71b8d5f129f99be20f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1554a13c60d54d71b8d5f129f99be20f 2024-12-11T20:34:34,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1554a13c60d54d71b8d5f129f99be20f, entries=150, sequenceid=483, filesize=12.0 K 2024-12-11T20:34:34,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0a19b15b87adee237235cecb1e4c99f in 224ms, sequenceid=483, compaction requested=true 2024-12-11T20:34:34,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:34,577 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:34,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:34,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:34,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:34,578 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:34,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:34,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:34,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:34,579 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:34,579 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/A is initiating minor compaction (all files) 2024-12-11T20:34:34,579 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:34,579 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/B is initiating minor compaction (all files) 2024-12-11T20:34:34,579 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/A in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,579 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/B in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,580 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/62b46ac1332043a89447249c916d792f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/54a7521be4dc44cdb1177e210db28718, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/4ed335ffecb3435da6ada2b4cafcbcec] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=39.4 K 2024-12-11T20:34:34,580 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b1438514cb324627ba34a954e3b553be, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8156ff0571734a7f8c5ec158de533880, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bddba88249154d43a38dbca1c3bfe1f3] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=37.0 K 2024-12-11T20:34:34,580 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 62b46ac1332043a89447249c916d792f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733949272776 2024-12-11T20:34:34,580 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1438514cb324627ba34a954e3b553be, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733949272776 2024-12-11T20:34:34,580 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 54a7521be4dc44cdb1177e210db28718, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733949273607 2024-12-11T20:34:34,581 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8156ff0571734a7f8c5ec158de533880, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733949273610 2024-12-11T20:34:34,581 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ed335ffecb3435da6ada2b4cafcbcec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949273640 2024-12-11T20:34:34,581 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting bddba88249154d43a38dbca1c3bfe1f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949273640 2024-12-11T20:34:34,592 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#A#compaction#98 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:34,593 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/68fab91f47be4aeeb24e7498642efd70 is 50, key is test_row_0/A:col10/1733949273640/Put/seqid=0 2024-12-11T20:34:34,594 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#B#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:34,595 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/9116a43e52e849b0b122d99264cfb1b4 is 50, key is test_row_0/B:col10/1733949273640/Put/seqid=0 2024-12-11T20:34:34,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741935_1111 (size=13425) 2024-12-11T20:34:34,635 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/68fab91f47be4aeeb24e7498642efd70 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/68fab91f47be4aeeb24e7498642efd70 2024-12-11T20:34:34,644 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/A of c0a19b15b87adee237235cecb1e4c99f into 68fab91f47be4aeeb24e7498642efd70(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:34,644 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:34,644 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/A, priority=13, startTime=1733949274577; duration=0sec 2024-12-11T20:34:34,645 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:34,645 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:34,645 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:34,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741936_1112 (size=13425) 2024-12-11T20:34:34,647 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:34,648 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/C is initiating minor compaction (all files) 2024-12-11T20:34:34,648 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/C in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,648 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/52bfa0d23a4b4ae5b7f90f23e47ee0ac, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/797428bde4724bd8bd747196ec37ed1d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1554a13c60d54d71b8d5f129f99be20f] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=37.0 K 2024-12-11T20:34:34,649 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 52bfa0d23a4b4ae5b7f90f23e47ee0ac, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1733949272776 2024-12-11T20:34:34,649 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 797428bde4724bd8bd747196ec37ed1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733949273610 2024-12-11T20:34:34,650 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 1554a13c60d54d71b8d5f129f99be20f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949273640 2024-12-11T20:34:34,654 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/9116a43e52e849b0b122d99264cfb1b4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/9116a43e52e849b0b122d99264cfb1b4 2024-12-11T20:34:34,663 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/B of c0a19b15b87adee237235cecb1e4c99f into 9116a43e52e849b0b122d99264cfb1b4(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:34,663 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:34,664 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/B, priority=13, startTime=1733949274578; duration=0sec 2024-12-11T20:34:34,664 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:34,664 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:34,668 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#C#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:34,669 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/3ec13714e57d48a7a42a4a81a3165da2 is 50, key is test_row_0/C:col10/1733949273640/Put/seqid=0 2024-12-11T20:34:34,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-11T20:34:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:34,671 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:34:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:34,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/2eb870bdbb0a4f83bcf6db2bb7a1f1f7 is 50, key is test_row_0/A:col10/1733949274412/Put/seqid=0 2024-12-11T20:34:34,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741938_1114 (size=12301) 2024-12-11T20:34:34,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741937_1113 (size=13391) 2024-12-11T20:34:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:34,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. as already flushing 2024-12-11T20:34:34,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949334822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:34,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:34,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949334926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:34,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:34,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T20:34:35,084 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/2eb870bdbb0a4f83bcf6db2bb7a1f1f7 2024-12-11T20:34:35,101 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/3ec13714e57d48a7a42a4a81a3165da2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/3ec13714e57d48a7a42a4a81a3165da2 2024-12-11T20:34:35,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/195a73005ead4f15975cdfdee8ed233b is 50, key is test_row_0/B:col10/1733949274412/Put/seqid=0 2024-12-11T20:34:35,113 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/C of c0a19b15b87adee237235cecb1e4c99f into 3ec13714e57d48a7a42a4a81a3165da2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:35,113 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:35,113 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/C, priority=13, startTime=1733949274578; duration=0sec 2024-12-11T20:34:35,113 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:35,113 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:35,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:35,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949335137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:35,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:35,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741939_1115 (size=12301) 2024-12-11T20:34:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949335457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:35,549 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/195a73005ead4f15975cdfdee8ed233b 2024-12-11T20:34:35,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/6be6f844e1c741f59c3dd01394cacf81 is 50, key is test_row_0/C:col10/1733949274412/Put/seqid=0 2024-12-11T20:34:35,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741940_1116 (size=12301) 2024-12-11T20:34:35,682 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-11T20:34:35,682 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:34:35,688 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-11T20:34:35,688 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:34:35,688 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:35,688 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:35,688 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-11T20:34:35,688 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:34:35,688 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:35,688 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:35,688 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:35,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:34:35,688 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:34:35,690 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:35,690 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:34:35,692 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-11T20:34:35,692 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:34:35,692 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:35,692 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:35,692 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:34:35,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:35,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56968 deadline: 1733949335977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:35,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:35,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:35,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:34:36,010 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/6be6f844e1c741f59c3dd01394cacf81 2024-12-11T20:34:36,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/2eb870bdbb0a4f83bcf6db2bb7a1f1f7 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/2eb870bdbb0a4f83bcf6db2bb7a1f1f7 2024-12-11T20:34:36,026 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/2eb870bdbb0a4f83bcf6db2bb7a1f1f7, entries=150, sequenceid=501, filesize=12.0 K 2024-12-11T20:34:36,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/195a73005ead4f15975cdfdee8ed233b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/195a73005ead4f15975cdfdee8ed233b 2024-12-11T20:34:36,035 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/195a73005ead4f15975cdfdee8ed233b, entries=150, sequenceid=501, filesize=12.0 K 2024-12-11T20:34:36,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/6be6f844e1c741f59c3dd01394cacf81 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6be6f844e1c741f59c3dd01394cacf81 2024-12-11T20:34:36,046 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6be6f844e1c741f59c3dd01394cacf81, entries=150, sequenceid=501, filesize=12.0 K 2024-12-11T20:34:36,047 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c0a19b15b87adee237235cecb1e4c99f in 1375ms, sequenceid=501, compaction requested=false 2024-12-11T20:34:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-11T20:34:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-11T20:34:36,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-11T20:34:36,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1480 sec 2024-12-11T20:34:36,055 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1550 sec 2024-12-11T20:34:36,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-11T20:34:36,058 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:36,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:36,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T20:34:36,999 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-11T20:34:36,999 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:34:36,999 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:36,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:37,000 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:37,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:37,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:37,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/3358054022f0430f9d96a02637a94fc4 is 50, key is test_row_0/A:col10/1733949274802/Put/seqid=0 2024-12-11T20:34:37,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741941_1117 (size=12301) 2024-12-11T20:34:37,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/3358054022f0430f9d96a02637a94fc4 2024-12-11T20:34:37,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/8cab05e92d8541c397b05d3671c4fe90 is 50, key is test_row_0/B:col10/1733949274802/Put/seqid=0 2024-12-11T20:34:37,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741942_1118 (size=12301) 2024-12-11T20:34:37,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/8cab05e92d8541c397b05d3671c4fe90 2024-12-11T20:34:37,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/c0de77923e3f47dba17074671304765e is 50, key is test_row_0/C:col10/1733949274802/Put/seqid=0 2024-12-11T20:34:37,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741943_1119 (size=12301) 2024-12-11T20:34:37,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/c0de77923e3f47dba17074671304765e 2024-12-11T20:34:37,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/3358054022f0430f9d96a02637a94fc4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/3358054022f0430f9d96a02637a94fc4 2024-12-11T20:34:37,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/3358054022f0430f9d96a02637a94fc4, entries=150, sequenceid=519, filesize=12.0 K 2024-12-11T20:34:37,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/8cab05e92d8541c397b05d3671c4fe90 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8cab05e92d8541c397b05d3671c4fe90 2024-12-11T20:34:37,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8cab05e92d8541c397b05d3671c4fe90, entries=150, sequenceid=519, filesize=12.0 K 2024-12-11T20:34:37,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/c0de77923e3f47dba17074671304765e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c0de77923e3f47dba17074671304765e 2024-12-11T20:34:37,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c0de77923e3f47dba17074671304765e, entries=150, sequenceid=519, filesize=12.0 K 2024-12-11T20:34:37,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for c0a19b15b87adee237235cecb1e4c99f in 82ms, sequenceid=519, compaction requested=true 2024-12-11T20:34:37,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:37,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:37,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:37,082 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:37,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:37,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:37,082 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:37,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0a19b15b87adee237235cecb1e4c99f:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:37,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:37,084 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:37,084 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/B is initiating minor compaction (all files) 2024-12-11T20:34:37,084 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/B in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:37,084 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/9116a43e52e849b0b122d99264cfb1b4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/195a73005ead4f15975cdfdee8ed233b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8cab05e92d8541c397b05d3671c4fe90] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=37.1 K 2024-12-11T20:34:37,085 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:37,085 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/A is initiating minor compaction (all files) 2024-12-11T20:34:37,085 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/A in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:37,085 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/68fab91f47be4aeeb24e7498642efd70, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/2eb870bdbb0a4f83bcf6db2bb7a1f1f7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/3358054022f0430f9d96a02637a94fc4] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=37.1 K 2024-12-11T20:34:37,086 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68fab91f47be4aeeb24e7498642efd70, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949273640 2024-12-11T20:34:37,086 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 9116a43e52e849b0b122d99264cfb1b4, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949273640 2024-12-11T20:34:37,086 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eb870bdbb0a4f83bcf6db2bb7a1f1f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733949274399 2024-12-11T20:34:37,087 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 195a73005ead4f15975cdfdee8ed233b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733949274399 2024-12-11T20:34:37,087 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3358054022f0430f9d96a02637a94fc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733949274802 2024-12-11T20:34:37,087 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cab05e92d8541c397b05d3671c4fe90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733949274802 2024-12-11T20:34:37,097 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#B#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:37,097 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/253bb5f14bd04bbe90d90d57e21f432d is 50, key is test_row_0/B:col10/1733949274802/Put/seqid=0 2024-12-11T20:34:37,105 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#A#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:37,105 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/6e0c8c7bfe1d4300aebdaa316d4d8cb8 is 50, key is test_row_0/A:col10/1733949274802/Put/seqid=0 2024-12-11T20:34:37,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741944_1120 (size=13527) 2024-12-11T20:34:37,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741945_1121 (size=13527) 2024-12-11T20:34:37,172 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T20:34:37,524 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/253bb5f14bd04bbe90d90d57e21f432d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/253bb5f14bd04bbe90d90d57e21f432d 2024-12-11T20:34:37,531 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/B of c0a19b15b87adee237235cecb1e4c99f into 253bb5f14bd04bbe90d90d57e21f432d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:37,531 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:37,531 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/B, priority=13, startTime=1733949277082; duration=0sec 2024-12-11T20:34:37,532 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:37,532 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:B 2024-12-11T20:34:37,532 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:37,534 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:37,534 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0a19b15b87adee237235cecb1e4c99f/C is initiating minor compaction (all files) 2024-12-11T20:34:37,534 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0a19b15b87adee237235cecb1e4c99f/C in TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:37,534 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/3ec13714e57d48a7a42a4a81a3165da2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6be6f844e1c741f59c3dd01394cacf81, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c0de77923e3f47dba17074671304765e] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp, totalSize=37.1 K 2024-12-11T20:34:37,535 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ec13714e57d48a7a42a4a81a3165da2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949273640 2024-12-11T20:34:37,536 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6be6f844e1c741f59c3dd01394cacf81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1733949274399 2024-12-11T20:34:37,536 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c0de77923e3f47dba17074671304765e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1733949274802 2024-12-11T20:34:37,536 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/6e0c8c7bfe1d4300aebdaa316d4d8cb8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/6e0c8c7bfe1d4300aebdaa316d4d8cb8 2024-12-11T20:34:37,543 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/A of c0a19b15b87adee237235cecb1e4c99f into 6e0c8c7bfe1d4300aebdaa316d4d8cb8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:37,543 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:37,543 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/A, priority=13, startTime=1733949277082; duration=0sec 2024-12-11T20:34:37,543 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:37,543 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:A 2024-12-11T20:34:37,546 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0a19b15b87adee237235cecb1e4c99f#C#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:37,547 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/8a8831cf4c664dee8c1601b791934cb8 is 50, key is test_row_0/C:col10/1733949274802/Put/seqid=0 2024-12-11T20:34:37,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741946_1122 (size=13493) 2024-12-11T20:34:37,959 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-11T20:34:37,959 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:34:37,959 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:37,959 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:37,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:34:37,968 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/8a8831cf4c664dee8c1601b791934cb8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/8a8831cf4c664dee8c1601b791934cb8 2024-12-11T20:34:37,970 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-11T20:34:37,970 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:34:37,970 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:37,970 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:37,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:34:37,979 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-11T20:34:37,979 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0a19b15b87adee237235cecb1e4c99f/C of c0a19b15b87adee237235cecb1e4c99f into 8a8831cf4c664dee8c1601b791934cb8(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:37,979 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:34:37,979 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0a19b15b87adee237235cecb1e4c99f: 2024-12-11T20:34:37,979 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:37,979 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., storeName=c0a19b15b87adee237235cecb1e4c99f/C, priority=13, startTime=1733949277082; duration=0sec 2024-12-11T20:34:37,979 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:37,979 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:34:37,979 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:37,979 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0a19b15b87adee237235cecb1e4c99f:C 2024-12-11T20:34:37,999 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-11T20:34:37,999 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:34:37,999 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:37,999 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:37,999 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:34:37,999 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T20:34:37,999 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 193 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3182 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3230 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1491 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4466 rows 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1475 2024-12-11T20:34:38,000 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4424 rows 2024-12-11T20:34:38,000 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T20:34:38,000 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:34:38,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:38,000 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:38,000 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:34:38,006 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-11T20:34:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T20:34:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T20:34:38,016 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949278015"}]},"ts":"1733949278015"} 2024-12-11T20:34:38,019 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T20:34:38,019 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T20:34:38,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T20:34:38,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0a19b15b87adee237235cecb1e4c99f, UNASSIGN}] 2024-12-11T20:34:38,028 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0a19b15b87adee237235cecb1e4c99f, UNASSIGN 2024-12-11T20:34:38,029 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=c0a19b15b87adee237235cecb1e4c99f, regionState=CLOSING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:38,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0a19b15b87adee237235cecb1e4c99f, UNASSIGN because future has completed 2024-12-11T20:34:38,033 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T20:34:38,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:34:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T20:34:38,191 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:38,191 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T20:34:38,192 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing c0a19b15b87adee237235cecb1e4c99f, disabling compactions & flushes 2024-12-11T20:34:38,192 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:38,192 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:38,192 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. after waiting 0 ms 2024-12-11T20:34:38,193 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:38,193 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing c0a19b15b87adee237235cecb1e4c99f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-11T20:34:38,193 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=A 2024-12-11T20:34:38,193 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:38,193 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=B 2024-12-11T20:34:38,193 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:38,193 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0a19b15b87adee237235cecb1e4c99f, store=C 2024-12-11T20:34:38,193 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:38,198 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/ce662d69d79b4895ab725b92f584ff56 is 50, key is test_row_1/A:col10/1733949277968/Put/seqid=0 2024-12-11T20:34:38,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741947_1123 (size=9857) 2024-12-11T20:34:38,229 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/ce662d69d79b4895ab725b92f584ff56 2024-12-11T20:34:38,244 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/e9aa2a2632e342188b3341f5bdf2a29d is 50, key is test_row_1/B:col10/1733949277968/Put/seqid=0 2024-12-11T20:34:38,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741948_1124 (size=9857) 2024-12-11T20:34:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T20:34:38,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T20:34:38,654 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/e9aa2a2632e342188b3341f5bdf2a29d 2024-12-11T20:34:38,662 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/844c11b2c0ed441abeffb0af1c3979b5 is 50, key is test_row_1/C:col10/1733949277968/Put/seqid=0 2024-12-11T20:34:38,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741949_1125 (size=9857) 2024-12-11T20:34:39,067 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/844c11b2c0ed441abeffb0af1c3979b5 2024-12-11T20:34:39,073 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/A/ce662d69d79b4895ab725b92f584ff56 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/ce662d69d79b4895ab725b92f584ff56 2024-12-11T20:34:39,079 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/ce662d69d79b4895ab725b92f584ff56, entries=100, sequenceid=529, filesize=9.6 K 2024-12-11T20:34:39,080 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/B/e9aa2a2632e342188b3341f5bdf2a29d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/e9aa2a2632e342188b3341f5bdf2a29d 2024-12-11T20:34:39,087 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/e9aa2a2632e342188b3341f5bdf2a29d, entries=100, sequenceid=529, filesize=9.6 K 2024-12-11T20:34:39,088 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/.tmp/C/844c11b2c0ed441abeffb0af1c3979b5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/844c11b2c0ed441abeffb0af1c3979b5 2024-12-11T20:34:39,094 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/844c11b2c0ed441abeffb0af1c3979b5, entries=100, sequenceid=529, filesize=9.6 K 2024-12-11T20:34:39,095 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c0a19b15b87adee237235cecb1e4c99f in 902ms, sequenceid=529, compaction requested=false 2024-12-11T20:34:39,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7b28ebbe7c0e45c1932093042e45b19c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/8efebfd0503b4522aec7116c3cb30207, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1531b5106fac48a6933ac88b02579d9f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/5b03928a41ca4bad9207a1c9e6955345, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/393de5cae4684c04835e553b24cf56a3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/01dfcdd03b7d4f5a8368a4825d2805e8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/cdcca54a28744e9cbf3d9f31abffbf51, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/76082bcb39b047aa93b1f36b2bda8ee7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/c05940b3ebda423ca6618949787e1785, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/328ac0fcd0cd4418b03ad1ece7c27ca6, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/70bbb5a438a24d7a822cee14b9986f8c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/ad88b8dbd9d84f71a2073a2088c0896c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/18c82d0a12b846c19ba2ec66f3acd28b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/d00563dd3a0744a4b88b3be1e4dac023, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/43afacfc9cca43c390d76326c6c55936, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/9128ac15b759492186b34a93d6bd4eb6, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/bdf5e78781c34a53b295901984fe574a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/228f67d04821478fab6490ea59836246, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/300f2d32ade447a2bb5df575b0ed22f2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/10a524e1e0924d34834259e50395a90a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/e2ba2b3f2bcf4d9aad566d7bd82ff7b1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/9418e44d81394567b872f4ba6fe7eaf2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/5c47876c87bb4082a9231c6ada150d95, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/356a54362f6f4f208e3f7d5a1d59e994, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7d7debc714724281bff9ae19f48b2b5e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/034b478569cf4dc0bea0e9980646ac0d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/80ed7d6634bb417896ed91637f3b9660, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/3e75c2d4504f410aa6b227d5e4df5161, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1cd47106850d46a0b4f9680d4688ef8f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/62b46ac1332043a89447249c916d792f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/14ba6017647a4a13a9d82cc28897c5de, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/54a7521be4dc44cdb1177e210db28718, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/68fab91f47be4aeeb24e7498642efd70, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/4ed335ffecb3435da6ada2b4cafcbcec, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/2eb870bdbb0a4f83bcf6db2bb7a1f1f7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/3358054022f0430f9d96a02637a94fc4] to archive 2024-12-11T20:34:39,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:34:39,109 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7b28ebbe7c0e45c1932093042e45b19c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7b28ebbe7c0e45c1932093042e45b19c 2024-12-11T20:34:39,109 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/393de5cae4684c04835e553b24cf56a3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/393de5cae4684c04835e553b24cf56a3 2024-12-11T20:34:39,109 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/01dfcdd03b7d4f5a8368a4825d2805e8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/01dfcdd03b7d4f5a8368a4825d2805e8 2024-12-11T20:34:39,109 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/5b03928a41ca4bad9207a1c9e6955345 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/5b03928a41ca4bad9207a1c9e6955345 2024-12-11T20:34:39,109 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/8efebfd0503b4522aec7116c3cb30207 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/8efebfd0503b4522aec7116c3cb30207 2024-12-11T20:34:39,109 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/cdcca54a28744e9cbf3d9f31abffbf51 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/cdcca54a28744e9cbf3d9f31abffbf51 2024-12-11T20:34:39,110 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/76082bcb39b047aa93b1f36b2bda8ee7 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/76082bcb39b047aa93b1f36b2bda8ee7 2024-12-11T20:34:39,110 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1531b5106fac48a6933ac88b02579d9f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1531b5106fac48a6933ac88b02579d9f 2024-12-11T20:34:39,111 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/70bbb5a438a24d7a822cee14b9986f8c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/70bbb5a438a24d7a822cee14b9986f8c 2024-12-11T20:34:39,111 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/18c82d0a12b846c19ba2ec66f3acd28b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/18c82d0a12b846c19ba2ec66f3acd28b 2024-12-11T20:34:39,112 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/ad88b8dbd9d84f71a2073a2088c0896c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/ad88b8dbd9d84f71a2073a2088c0896c 2024-12-11T20:34:39,112 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/c05940b3ebda423ca6618949787e1785 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/c05940b3ebda423ca6618949787e1785 2024-12-11T20:34:39,113 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/d00563dd3a0744a4b88b3be1e4dac023 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/d00563dd3a0744a4b88b3be1e4dac023 2024-12-11T20:34:39,113 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/43afacfc9cca43c390d76326c6c55936 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/43afacfc9cca43c390d76326c6c55936 2024-12-11T20:34:39,113 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/328ac0fcd0cd4418b03ad1ece7c27ca6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/328ac0fcd0cd4418b03ad1ece7c27ca6 2024-12-11T20:34:39,114 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/9128ac15b759492186b34a93d6bd4eb6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/9128ac15b759492186b34a93d6bd4eb6 2024-12-11T20:34:39,115 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/228f67d04821478fab6490ea59836246 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/228f67d04821478fab6490ea59836246 2024-12-11T20:34:39,115 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/300f2d32ade447a2bb5df575b0ed22f2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/300f2d32ade447a2bb5df575b0ed22f2 2024-12-11T20:34:39,115 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/e2ba2b3f2bcf4d9aad566d7bd82ff7b1 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/e2ba2b3f2bcf4d9aad566d7bd82ff7b1 2024-12-11T20:34:39,115 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/5c47876c87bb4082a9231c6ada150d95 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/5c47876c87bb4082a9231c6ada150d95 2024-12-11T20:34:39,115 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/bdf5e78781c34a53b295901984fe574a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/bdf5e78781c34a53b295901984fe574a 2024-12-11T20:34:39,116 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/10a524e1e0924d34834259e50395a90a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/10a524e1e0924d34834259e50395a90a 2024-12-11T20:34:39,116 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/356a54362f6f4f208e3f7d5a1d59e994 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/356a54362f6f4f208e3f7d5a1d59e994 2024-12-11T20:34:39,116 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/9418e44d81394567b872f4ba6fe7eaf2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/9418e44d81394567b872f4ba6fe7eaf2 2024-12-11T20:34:39,117 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7d7debc714724281bff9ae19f48b2b5e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/7d7debc714724281bff9ae19f48b2b5e 2024-12-11T20:34:39,117 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/034b478569cf4dc0bea0e9980646ac0d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/034b478569cf4dc0bea0e9980646ac0d 2024-12-11T20:34:39,117 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/80ed7d6634bb417896ed91637f3b9660 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/80ed7d6634bb417896ed91637f3b9660 2024-12-11T20:34:39,118 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1cd47106850d46a0b4f9680d4688ef8f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/1cd47106850d46a0b4f9680d4688ef8f 2024-12-11T20:34:39,118 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/14ba6017647a4a13a9d82cc28897c5de to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/14ba6017647a4a13a9d82cc28897c5de 2024-12-11T20:34:39,118 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/3e75c2d4504f410aa6b227d5e4df5161 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/3e75c2d4504f410aa6b227d5e4df5161 2024-12-11T20:34:39,119 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/62b46ac1332043a89447249c916d792f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/62b46ac1332043a89447249c916d792f 2024-12-11T20:34:39,120 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/54a7521be4dc44cdb1177e210db28718 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/54a7521be4dc44cdb1177e210db28718 2024-12-11T20:34:39,120 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/68fab91f47be4aeeb24e7498642efd70 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/68fab91f47be4aeeb24e7498642efd70 2024-12-11T20:34:39,121 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/3358054022f0430f9d96a02637a94fc4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/3358054022f0430f9d96a02637a94fc4 2024-12-11T20:34:39,121 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/2eb870bdbb0a4f83bcf6db2bb7a1f1f7 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/2eb870bdbb0a4f83bcf6db2bb7a1f1f7 2024-12-11T20:34:39,121 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/4ed335ffecb3435da6ada2b4cafcbcec to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/4ed335ffecb3435da6ada2b4cafcbcec 2024-12-11T20:34:39,137 DEBUG [StoreCloser-TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/48c8b56b0e6845c385cb31f03f9ac6af, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6871865fc1384259965df21b1986ce11, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/51de29dcb3324c01a0266ec03f3b5e85, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ae950731eb64b6fbfe181024f8bf131, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b44fbaf0f514458d968f2023ac6368b3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/3127b3de0fc04f3badf9d50245713167, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/043b917cf31d40d4836187df87687fa8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/35e3ff6660594b3e8561e4ff2c3efefc, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/dc704688418a4408876304a0259d1314, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f4c73fb614ee4e45b4f2a672559142b7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bfef54fa90aa48a7836fbffb4699271a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d9a3bcc7805e4e928bb2eb4640942afb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/33478ca0a3a44dbebfff6e45e3a55e41, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/a2467b7b59c94b00a1bc9bd5bfe9b32e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/0b2e1f47777b458296b77f2645f05c81, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b5878fde768e4becaed9ae662ba216ff, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b7af5517a7014912bc237289b0b4e82a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d633b9a20f44491cbc8cb819c99b853c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/c87fcc961186489dbe1507c449fe8787, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/4ffec81a4a3e4ea08b7f23791f19404c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6ce531b6aa03435f890b4fa26d443677, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/de25570c454849d49a3262f0b50c0851, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6c1aad6406b444268ac657f13904cb65, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/07c6932d51b34a6ea4283afed6e0653b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/c00a100c016f426f8b4ff47a1cd6ea45, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ffc0dd14adc4ea5b2a5fd4222f35759, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d2118cb60e9b4894995ae16db664cbbc, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f87c9a06f1fd47d3b17f6d93ecd14f8c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/fbb61589f4b24ed19f991387c68b0d48, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b1438514cb324627ba34a954e3b553be, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/613744e9b11a41d987af770dc73b61c8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8156ff0571734a7f8c5ec158de533880, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/9116a43e52e849b0b122d99264cfb1b4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bddba88249154d43a38dbca1c3bfe1f3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/195a73005ead4f15975cdfdee8ed233b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8cab05e92d8541c397b05d3671c4fe90] to archive 2024-12-11T20:34:39,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:34:39,141 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/3127b3de0fc04f3badf9d50245713167 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/3127b3de0fc04f3badf9d50245713167 2024-12-11T20:34:39,141 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6871865fc1384259965df21b1986ce11 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6871865fc1384259965df21b1986ce11 2024-12-11T20:34:39,141 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/35e3ff6660594b3e8561e4ff2c3efefc to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/35e3ff6660594b3e8561e4ff2c3efefc 2024-12-11T20:34:39,141 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/043b917cf31d40d4836187df87687fa8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/043b917cf31d40d4836187df87687fa8 2024-12-11T20:34:39,142 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/51de29dcb3324c01a0266ec03f3b5e85 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/51de29dcb3324c01a0266ec03f3b5e85 2024-12-11T20:34:39,142 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/48c8b56b0e6845c385cb31f03f9ac6af to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/48c8b56b0e6845c385cb31f03f9ac6af 2024-12-11T20:34:39,142 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ae950731eb64b6fbfe181024f8bf131 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ae950731eb64b6fbfe181024f8bf131 2024-12-11T20:34:39,142 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b44fbaf0f514458d968f2023ac6368b3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b44fbaf0f514458d968f2023ac6368b3 2024-12-11T20:34:39,144 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bfef54fa90aa48a7836fbffb4699271a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bfef54fa90aa48a7836fbffb4699271a 2024-12-11T20:34:39,144 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f4c73fb614ee4e45b4f2a672559142b7 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f4c73fb614ee4e45b4f2a672559142b7 2024-12-11T20:34:39,144 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/dc704688418a4408876304a0259d1314 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/dc704688418a4408876304a0259d1314 2024-12-11T20:34:39,145 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d9a3bcc7805e4e928bb2eb4640942afb to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d9a3bcc7805e4e928bb2eb4640942afb 2024-12-11T20:34:39,145 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/33478ca0a3a44dbebfff6e45e3a55e41 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/33478ca0a3a44dbebfff6e45e3a55e41 2024-12-11T20:34:39,145 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/a2467b7b59c94b00a1bc9bd5bfe9b32e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/a2467b7b59c94b00a1bc9bd5bfe9b32e 2024-12-11T20:34:39,146 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/0b2e1f47777b458296b77f2645f05c81 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/0b2e1f47777b458296b77f2645f05c81 2024-12-11T20:34:39,146 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b5878fde768e4becaed9ae662ba216ff to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b5878fde768e4becaed9ae662ba216ff 2024-12-11T20:34:39,146 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b7af5517a7014912bc237289b0b4e82a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b7af5517a7014912bc237289b0b4e82a 2024-12-11T20:34:39,147 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/c87fcc961186489dbe1507c449fe8787 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/c87fcc961186489dbe1507c449fe8787 2024-12-11T20:34:39,147 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d633b9a20f44491cbc8cb819c99b853c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d633b9a20f44491cbc8cb819c99b853c 2024-12-11T20:34:39,147 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/de25570c454849d49a3262f0b50c0851 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/de25570c454849d49a3262f0b50c0851 2024-12-11T20:34:39,148 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6ce531b6aa03435f890b4fa26d443677 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6ce531b6aa03435f890b4fa26d443677 2024-12-11T20:34:39,148 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/c00a100c016f426f8b4ff47a1cd6ea45 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/c00a100c016f426f8b4ff47a1cd6ea45 2024-12-11T20:34:39,149 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/4ffec81a4a3e4ea08b7f23791f19404c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/4ffec81a4a3e4ea08b7f23791f19404c 2024-12-11T20:34:39,149 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ffc0dd14adc4ea5b2a5fd4222f35759 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/7ffc0dd14adc4ea5b2a5fd4222f35759 2024-12-11T20:34:39,149 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6c1aad6406b444268ac657f13904cb65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/6c1aad6406b444268ac657f13904cb65 2024-12-11T20:34:39,149 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/07c6932d51b34a6ea4283afed6e0653b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/07c6932d51b34a6ea4283afed6e0653b 2024-12-11T20:34:39,150 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f87c9a06f1fd47d3b17f6d93ecd14f8c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/f87c9a06f1fd47d3b17f6d93ecd14f8c 2024-12-11T20:34:39,150 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d2118cb60e9b4894995ae16db664cbbc to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/d2118cb60e9b4894995ae16db664cbbc 2024-12-11T20:34:39,151 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/fbb61589f4b24ed19f991387c68b0d48 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/fbb61589f4b24ed19f991387c68b0d48 2024-12-11T20:34:39,151 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b1438514cb324627ba34a954e3b553be to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/b1438514cb324627ba34a954e3b553be 2024-12-11T20:34:39,151 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8156ff0571734a7f8c5ec158de533880 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8156ff0571734a7f8c5ec158de533880 2024-12-11T20:34:39,151 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/9116a43e52e849b0b122d99264cfb1b4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/9116a43e52e849b0b122d99264cfb1b4 2024-12-11T20:34:39,151 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bddba88249154d43a38dbca1c3bfe1f3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/bddba88249154d43a38dbca1c3bfe1f3 2024-12-11T20:34:39,151 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/613744e9b11a41d987af770dc73b61c8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/613744e9b11a41d987af770dc73b61c8 2024-12-11T20:34:39,152 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/195a73005ead4f15975cdfdee8ed233b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/195a73005ead4f15975cdfdee8ed233b 2024-12-11T20:34:39,152 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8cab05e92d8541c397b05d3671c4fe90 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/8cab05e92d8541c397b05d3671c4fe90 2024-12-11T20:34:39,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d5b33b1feec74b23bdb97066111b5e6e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6e968282688a4016b4195a4ec9fa3dab, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/afd1bd108e3f4c7aa6564168ede4f6c0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/3d7bef8e38ab4aab888905eb86328239, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1dbd509fcee3445bb214e6877cb78310, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/db2a7804a33c43cbb0ef79f430316b59, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/aac9b1ac6b6b4f49b6620a8cfcd90f46, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/9b8fd8b2110b4024bf54b244513e4a2e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d1133f81671b4c78b7f43e64b4acda20, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6d18bf8718a04027ab4f52402586285c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/f88ccef3c1e948149d28700fda235bd4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/324c392b35544801ac3c59725361de22, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/87b67812b3274ba3b6031ab26baba174, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/2a9209ce12264f1699c488d6ed6b566b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/961357cb33c54703bda2fcf2da0ea372, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d18795d22ef044b598a5274325339994, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/94b793d94b3c4a85a654fc5bd896b160, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/69cd5f4edb2c4517ad9e985332b38bcd, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/af11fbd239944f3688a64f03607b3e38, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/7e0260a94ce34b58a095a9f20c7bf5df, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/4e7df3410361438aa2b7fddb89d3d4ab, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/212aa9c0243f485b97201608e743bf46, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1d8d495ad414465c8b11b9a527a26788, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/a0ae5c74fb4c4345908c0e2680ed2136, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/411b18d9fca44904a3c2057256297e06, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d1444fdbb8634518b373945ed7acda4e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/8116a2d8432f4771b690865313fbb98b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/78dd05ea840943888b39ae23d6275098, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/52bfa0d23a4b4ae5b7f90f23e47ee0ac, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c965a82bec304bc18fe566e3586a2102, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/797428bde4724bd8bd747196ec37ed1d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/3ec13714e57d48a7a42a4a81a3165da2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1554a13c60d54d71b8d5f129f99be20f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6be6f844e1c741f59c3dd01394cacf81, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c0de77923e3f47dba17074671304765e] to archive 2024-12-11T20:34:39,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:34:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T20:34:39,157 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d5b33b1feec74b23bdb97066111b5e6e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d5b33b1feec74b23bdb97066111b5e6e 2024-12-11T20:34:39,158 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/db2a7804a33c43cbb0ef79f430316b59 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/db2a7804a33c43cbb0ef79f430316b59 2024-12-11T20:34:39,158 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/aac9b1ac6b6b4f49b6620a8cfcd90f46 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/aac9b1ac6b6b4f49b6620a8cfcd90f46 2024-12-11T20:34:39,158 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1dbd509fcee3445bb214e6877cb78310 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1dbd509fcee3445bb214e6877cb78310 2024-12-11T20:34:39,158 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/3d7bef8e38ab4aab888905eb86328239 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/3d7bef8e38ab4aab888905eb86328239 2024-12-11T20:34:39,158 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6e968282688a4016b4195a4ec9fa3dab to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6e968282688a4016b4195a4ec9fa3dab 2024-12-11T20:34:39,158 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/9b8fd8b2110b4024bf54b244513e4a2e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/9b8fd8b2110b4024bf54b244513e4a2e 2024-12-11T20:34:39,159 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/afd1bd108e3f4c7aa6564168ede4f6c0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/afd1bd108e3f4c7aa6564168ede4f6c0 2024-12-11T20:34:39,160 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/f88ccef3c1e948149d28700fda235bd4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/f88ccef3c1e948149d28700fda235bd4 2024-12-11T20:34:39,163 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d1133f81671b4c78b7f43e64b4acda20 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d1133f81671b4c78b7f43e64b4acda20 2024-12-11T20:34:39,163 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6d18bf8718a04027ab4f52402586285c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6d18bf8718a04027ab4f52402586285c 2024-12-11T20:34:39,163 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/87b67812b3274ba3b6031ab26baba174 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/87b67812b3274ba3b6031ab26baba174 2024-12-11T20:34:39,164 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/324c392b35544801ac3c59725361de22 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/324c392b35544801ac3c59725361de22 2024-12-11T20:34:39,164 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/2a9209ce12264f1699c488d6ed6b566b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/2a9209ce12264f1699c488d6ed6b566b 2024-12-11T20:34:39,164 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d18795d22ef044b598a5274325339994 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d18795d22ef044b598a5274325339994 2024-12-11T20:34:39,165 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/94b793d94b3c4a85a654fc5bd896b160 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/94b793d94b3c4a85a654fc5bd896b160 2024-12-11T20:34:39,166 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/69cd5f4edb2c4517ad9e985332b38bcd to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/69cd5f4edb2c4517ad9e985332b38bcd 2024-12-11T20:34:39,166 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/7e0260a94ce34b58a095a9f20c7bf5df to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/7e0260a94ce34b58a095a9f20c7bf5df 2024-12-11T20:34:39,166 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/212aa9c0243f485b97201608e743bf46 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/212aa9c0243f485b97201608e743bf46 2024-12-11T20:34:39,166 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1d8d495ad414465c8b11b9a527a26788 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1d8d495ad414465c8b11b9a527a26788 2024-12-11T20:34:39,167 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/af11fbd239944f3688a64f03607b3e38 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/af11fbd239944f3688a64f03607b3e38 2024-12-11T20:34:39,168 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/961357cb33c54703bda2fcf2da0ea372 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/961357cb33c54703bda2fcf2da0ea372 2024-12-11T20:34:39,168 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d1444fdbb8634518b373945ed7acda4e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/d1444fdbb8634518b373945ed7acda4e 2024-12-11T20:34:39,168 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/411b18d9fca44904a3c2057256297e06 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/411b18d9fca44904a3c2057256297e06 2024-12-11T20:34:39,168 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/78dd05ea840943888b39ae23d6275098 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/78dd05ea840943888b39ae23d6275098 2024-12-11T20:34:39,168 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/a0ae5c74fb4c4345908c0e2680ed2136 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/a0ae5c74fb4c4345908c0e2680ed2136 2024-12-11T20:34:39,168 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/8116a2d8432f4771b690865313fbb98b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/8116a2d8432f4771b690865313fbb98b 2024-12-11T20:34:39,170 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/52bfa0d23a4b4ae5b7f90f23e47ee0ac to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/52bfa0d23a4b4ae5b7f90f23e47ee0ac 2024-12-11T20:34:39,170 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/797428bde4724bd8bd747196ec37ed1d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/797428bde4724bd8bd747196ec37ed1d 2024-12-11T20:34:39,170 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1554a13c60d54d71b8d5f129f99be20f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/1554a13c60d54d71b8d5f129f99be20f 2024-12-11T20:34:39,170 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c0de77923e3f47dba17074671304765e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c0de77923e3f47dba17074671304765e 2024-12-11T20:34:39,170 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c965a82bec304bc18fe566e3586a2102 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/c965a82bec304bc18fe566e3586a2102 2024-12-11T20:34:39,170 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/3ec13714e57d48a7a42a4a81a3165da2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/3ec13714e57d48a7a42a4a81a3165da2 2024-12-11T20:34:39,170 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6be6f844e1c741f59c3dd01394cacf81 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/6be6f844e1c741f59c3dd01394cacf81 2024-12-11T20:34:39,171 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/4e7df3410361438aa2b7fddb89d3d4ab to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/4e7df3410361438aa2b7fddb89d3d4ab 2024-12-11T20:34:39,184 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/recovered.edits/532.seqid, newMaxSeqId=532, maxSeqId=1 2024-12-11T20:34:39,186 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. 2024-12-11T20:34:39,187 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for c0a19b15b87adee237235cecb1e4c99f: Waiting for close lock at 1733949278192Running coprocessor pre-close hooks at 1733949278192Disabling compacts and flushes for region at 1733949278192Disabling writes for close at 1733949278193 (+1 ms)Obtaining lock to block concurrent updates at 1733949278193Preparing flush snapshotting stores in c0a19b15b87adee237235cecb1e4c99f at 1733949278193Finished memstore snapshotting TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733949278193Flushing stores of TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f. at 1733949278194 (+1 ms)Flushing c0a19b15b87adee237235cecb1e4c99f/A: creating writer at 1733949278194Flushing c0a19b15b87adee237235cecb1e4c99f/A: appending metadata at 1733949278197 (+3 ms)Flushing c0a19b15b87adee237235cecb1e4c99f/A: closing flushed file at 1733949278197Flushing c0a19b15b87adee237235cecb1e4c99f/B: creating writer at 1733949278235 (+38 ms)Flushing c0a19b15b87adee237235cecb1e4c99f/B: appending metadata at 1733949278244 (+9 ms)Flushing c0a19b15b87adee237235cecb1e4c99f/B: closing flushed file at 1733949278244Flushing c0a19b15b87adee237235cecb1e4c99f/C: creating writer at 1733949278659 (+415 ms)Flushing c0a19b15b87adee237235cecb1e4c99f/C: appending metadata at 1733949278662 (+3 ms)Flushing c0a19b15b87adee237235cecb1e4c99f/C: closing flushed file at 1733949278662Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ba77523: reopening flushed file at 1733949279072 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@262cd315: reopening flushed file at 1733949279079 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70eb7776: reopening flushed file at 1733949279087 (+8 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c0a19b15b87adee237235cecb1e4c99f in 902ms, sequenceid=529, compaction requested=false at 1733949279095 (+8 ms)Writing region close event to WAL at 1733949279175 (+80 ms)Running coprocessor post-close hooks at 1733949279184 (+9 ms)Closed at 1733949279186 (+2 ms) 2024-12-11T20:34:39,190 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:39,190 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=c0a19b15b87adee237235cecb1e4c99f, regionState=CLOSED 2024-12-11T20:34:39,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:34:39,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-11T20:34:39,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure c0a19b15b87adee237235cecb1e4c99f, server=5be53b084ac7,44543,1733949248752 in 1.1610 sec 2024-12-11T20:34:39,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-11T20:34:39,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0a19b15b87adee237235cecb1e4c99f, UNASSIGN in 1.1710 sec 2024-12-11T20:34:39,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-11T20:34:39,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1790 sec 2024-12-11T20:34:39,203 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949279203"}]},"ts":"1733949279203"} 2024-12-11T20:34:39,205 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T20:34:39,205 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T20:34:39,207 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1980 sec 2024-12-11T20:34:40,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-11T20:34:40,168 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:40,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-11T20:34:40,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:34:40,180 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:34:40,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-11T20:34:40,182 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:34:40,191 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:40,196 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/recovered.edits] 2024-12-11T20:34:40,199 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/ce662d69d79b4895ab725b92f584ff56 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/ce662d69d79b4895ab725b92f584ff56 2024-12-11T20:34:40,200 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/6e0c8c7bfe1d4300aebdaa316d4d8cb8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/A/6e0c8c7bfe1d4300aebdaa316d4d8cb8 2024-12-11T20:34:40,203 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/253bb5f14bd04bbe90d90d57e21f432d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/253bb5f14bd04bbe90d90d57e21f432d 2024-12-11T20:34:40,203 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/e9aa2a2632e342188b3341f5bdf2a29d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/B/e9aa2a2632e342188b3341f5bdf2a29d 2024-12-11T20:34:40,216 DEBUG [HFileArchiver-8 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/844c11b2c0ed441abeffb0af1c3979b5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/844c11b2c0ed441abeffb0af1c3979b5 2024-12-11T20:34:40,216 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/8a8831cf4c664dee8c1601b791934cb8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/C/8a8831cf4c664dee8c1601b791934cb8 2024-12-11T20:34:40,220 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/recovered.edits/532.seqid to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f/recovered.edits/532.seqid 2024-12-11T20:34:40,220 DEBUG [HFileArchiver-7 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0a19b15b87adee237235cecb1e4c99f 2024-12-11T20:34:40,220 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T20:34:40,227 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:34:40,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-11T20:34:40,243 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T20:34:40,284 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T20:34:40,286 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:34:40,286 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T20:34:40,287 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733949280287"}]},"ts":"9223372036854775807"} 2024-12-11T20:34:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-11T20:34:40,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T20:34:40,293 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c0a19b15b87adee237235cecb1e4c99f, NAME => 'TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T20:34:40,293 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T20:34:40,293 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733949280293"}]},"ts":"9223372036854775807"} 2024-12-11T20:34:40,297 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T20:34:40,298 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:34:40,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 124 msec 2024-12-11T20:34:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-11T20:34:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T20:34:40,498 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:40,515 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=257 (was 211) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-724065718_22 at /127.0.0.1:54264 [Waiting for operation #338] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1936318462_22 at /127.0.0.1:39226 [Waiting for operation #353] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1936318462_22 at /127.0.0.1:52720 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;5be53b084ac7:44543-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-724065718_22 at /127.0.0.1:52726 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=400 (was 183) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2166 (was 2876) 2024-12-11T20:34:40,527 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=257, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=400, ProcessCount=11, AvailableMemoryMB=2164 2024-12-11T20:34:40,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T20:34:40,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:34:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T20:34:40,533 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:34:40,534 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:40,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-12-11T20:34:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-11T20:34:40,538 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:34:40,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741950_1126 (size=1061) 2024-12-11T20:34:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-11T20:34:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-11T20:34:40,964 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043 2024-12-11T20:34:40,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741951_1127 (size=53) 2024-12-11T20:34:40,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:34:40,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 786cab87062764a85ac24e1e3653be4a, disabling compactions & flushes 2024-12-11T20:34:40,976 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:40,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:40,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. after waiting 0 ms 2024-12-11T20:34:40,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:40,976 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:40,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 786cab87062764a85ac24e1e3653be4a: Waiting for close lock at 1733949280976Disabling compacts and flushes for region at 1733949280976Disabling writes for close at 1733949280976Writing region close event to WAL at 1733949280976Closed at 1733949280976 2024-12-11T20:34:40,978 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:34:40,978 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733949280978"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733949280978"}]},"ts":"1733949280978"} 2024-12-11T20:34:40,981 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:34:40,982 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:34:40,982 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949280982"}]},"ts":"1733949280982"} 2024-12-11T20:34:40,984 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T20:34:40,985 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, ASSIGN}] 2024-12-11T20:34:40,986 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, ASSIGN 2024-12-11T20:34:40,987 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, ASSIGN; state=OFFLINE, location=5be53b084ac7,44543,1733949248752; forceNewPlan=false, retain=false 2024-12-11T20:34:41,138 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=786cab87062764a85ac24e1e3653be4a, regionState=OPENING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:41,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, ASSIGN because future has completed 2024-12-11T20:34:41,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:34:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-11T20:34:41,297 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:41,297 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:34:41,298 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,298 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:34:41,298 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,298 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,300 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,301 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:34:41,302 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 786cab87062764a85ac24e1e3653be4a columnFamilyName A 2024-12-11T20:34:41,303 DEBUG [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:41,303 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(327): Store=786cab87062764a85ac24e1e3653be4a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:41,304 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,305 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:34:41,306 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 786cab87062764a85ac24e1e3653be4a columnFamilyName B 2024-12-11T20:34:41,306 DEBUG [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:41,306 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(327): Store=786cab87062764a85ac24e1e3653be4a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:41,306 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,308 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:34:41,308 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 786cab87062764a85ac24e1e3653be4a columnFamilyName C 2024-12-11T20:34:41,308 DEBUG [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:41,309 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(327): Store=786cab87062764a85ac24e1e3653be4a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:41,309 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:41,310 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,311 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,311 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,312 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,312 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,313 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T20:34:41,314 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,318 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:34:41,318 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 786cab87062764a85ac24e1e3653be4a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69844897, jitterRate=0.040770068764686584}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T20:34:41,319 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,320 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 786cab87062764a85ac24e1e3653be4a: Running coprocessor pre-open hook at 1733949281298Writing region info on filesystem at 1733949281298Initializing all the Stores at 1733949281299 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949281299Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949281300 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949281300Cleaning up temporary data from old regions at 1733949281312 (+12 ms)Running coprocessor post-open hooks at 1733949281319 (+7 ms)Region opened successfully at 1733949281320 (+1 ms) 2024-12-11T20:34:41,321 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., pid=32, masterSystemTime=1733949281294 2024-12-11T20:34:41,324 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:41,324 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:41,324 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=786cab87062764a85ac24e1e3653be4a, regionState=OPEN, openSeqNum=2, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:41,326 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:34:41,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-11T20:34:41,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 in 186 msec 2024-12-11T20:34:41,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-11T20:34:41,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, ASSIGN in 346 msec 2024-12-11T20:34:41,333 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:34:41,333 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949281333"}]},"ts":"1733949281333"} 2024-12-11T20:34:41,336 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T20:34:41,338 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:34:41,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 809 msec 2024-12-11T20:34:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-12-11T20:34:41,677 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:41,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@438596fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:41,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:41,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:41,680 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73af3d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:41,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:41,682 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43172, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3df773, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:41,684 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:41,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:41,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49700, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:41,689 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T20:34:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 5be53b084ac7,39923,1733949248041 2024-12-11T20:34:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44e20f97 2024-12-11T20:34:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T20:34:41,691 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T20:34:41,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T20:34:41,696 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:34:41,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-11T20:34:41,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741952_1128 (size=1097) 2024-12-11T20:34:41,721 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-11T20:34:41,721 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-11T20:34:41,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T20:34:41,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, REOPEN/MOVE}] 2024-12-11T20:34:41,734 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, REOPEN/MOVE 2024-12-11T20:34:41,735 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=786cab87062764a85ac24e1e3653be4a, regionState=CLOSING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:41,736 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, REOPEN/MOVE because future has completed 2024-12-11T20:34:41,737 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T20:34:41,737 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:34:41,890 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,890 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T20:34:41,890 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 786cab87062764a85ac24e1e3653be4a, disabling compactions & flushes 2024-12-11T20:34:41,890 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:41,891 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:41,891 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. after waiting 0 ms 2024-12-11T20:34:41,891 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:41,895 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T20:34:41,896 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:41,896 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 786cab87062764a85ac24e1e3653be4a: Waiting for close lock at 1733949281890Running coprocessor pre-close hooks at 1733949281890Disabling compacts and flushes for region at 1733949281890Disabling writes for close at 1733949281891 (+1 ms)Writing region close event to WAL at 1733949281891Running coprocessor post-close hooks at 1733949281896 (+5 ms)Closed at 1733949281896 2024-12-11T20:34:41,896 WARN [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 786cab87062764a85ac24e1e3653be4a to self. 2024-12-11T20:34:41,899 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:41,900 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=786cab87062764a85ac24e1e3653be4a, regionState=CLOSED 2024-12-11T20:34:41,902 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:34:41,906 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-11T20:34:41,907 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 in 167 msec 2024-12-11T20:34:41,907 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, REOPEN/MOVE; state=CLOSED, location=5be53b084ac7,44543,1733949248752; forceNewPlan=false, retain=true 2024-12-11T20:34:42,058 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=786cab87062764a85ac24e1e3653be4a, regionState=OPENING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,061 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, REOPEN/MOVE because future has completed 2024-12-11T20:34:42,061 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:34:42,218 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,218 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:34:42,218 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,219 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:34:42,219 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,219 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,222 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,224 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:34:42,232 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 786cab87062764a85ac24e1e3653be4a columnFamilyName A 2024-12-11T20:34:42,234 DEBUG [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:42,235 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(327): Store=786cab87062764a85ac24e1e3653be4a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:42,236 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,237 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:34:42,237 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 786cab87062764a85ac24e1e3653be4a columnFamilyName B 2024-12-11T20:34:42,237 DEBUG [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:42,238 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(327): Store=786cab87062764a85ac24e1e3653be4a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:42,238 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,239 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:34:42,239 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 786cab87062764a85ac24e1e3653be4a columnFamilyName C 2024-12-11T20:34:42,239 DEBUG [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:42,240 INFO [StoreOpener-786cab87062764a85ac24e1e3653be4a-1 {}] regionserver.HStore(327): Store=786cab87062764a85ac24e1e3653be4a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:34:42,240 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,240 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,241 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,243 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,244 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,244 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,245 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T20:34:42,247 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,248 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 786cab87062764a85ac24e1e3653be4a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65716839, jitterRate=-0.020742788910865784}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T20:34:42,248 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,249 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 786cab87062764a85ac24e1e3653be4a: Running coprocessor pre-open hook at 1733949282219Writing region info on filesystem at 1733949282219Initializing all the Stores at 1733949282220 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949282220Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949282220Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949282220Cleaning up temporary data from old regions at 1733949282244 (+24 ms)Running coprocessor post-open hooks at 1733949282248 (+4 ms)Region opened successfully at 1733949282249 (+1 ms) 2024-12-11T20:34:42,250 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., pid=37, masterSystemTime=1733949282214 2024-12-11T20:34:42,253 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,253 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,254 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=786cab87062764a85ac24e1e3653be4a, regionState=OPEN, openSeqNum=5, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:34:42,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-12-11T20:34:42,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 in 198 msec 2024-12-11T20:34:42,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-11T20:34:42,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, REOPEN/MOVE in 529 msec 2024-12-11T20:34:42,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-12-11T20:34:42,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 543 msec 2024-12-11T20:34:42,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 575 msec 2024-12-11T20:34:42,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-11T20:34:42,279 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:42,280 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57c57616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,280 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:42,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:42,281 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e1ef6f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,282 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:42,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:42,284 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:42,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e16802, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fb567f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:42,286 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:42,287 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:42,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e246ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:42,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:42,289 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:42,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391333b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42afb046, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,292 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:42,292 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:42,293 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:42,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bf96193, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:42,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:42,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:42,296 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:42,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295bf609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,298 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf72dc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,298 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:42,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:42,300 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a0a6bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:42,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:42,302 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43258, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:42,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f6fcd54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648e3749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:42,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:42,305 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:42,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:42,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:42,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e051b94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,307 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:42,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:42,309 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:42,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4ec09d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d2a07ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:42,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:42,312 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:42,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26408ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:42,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:42,315 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:42,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2597c875, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391c59ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:42,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:42,319 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fb22a63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:42,321 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:42,323 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ed7830, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e7c72d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:42,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:42,326 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:42,326 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eadcc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:42,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:42,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:42,329 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:42,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c0eac94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6095c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:34:42,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:34:42,332 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:34:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:34:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:34:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@304ab603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,333 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:34:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:34:42,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:34:42,335 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:34:42,336 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6df162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:34:42,339 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:42,340 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:42,341 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:42,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:42,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:42,343 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49720, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:42,343 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:42,344 DEBUG [Thread-642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:42,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:42,346 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:42,349 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:34:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:34:42,352 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:42,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:34:42,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:34:42,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:34:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-11T20:34:42,364 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T20:34:42,365 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:42,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:42,371 DEBUG [Thread-657 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:42,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:42,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:42,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:42,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:34:42,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:42,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:42,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:42,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:42,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:42,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:42,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:34:42,390 DEBUG [Thread-644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:42,391 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:42,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:42,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:42,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:42,395 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:42,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:34:42,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:34:42,410 DEBUG [Thread-655 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:34:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:34:42,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:34:42,413 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:34:42,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:34:42,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211b998f90688f94e98aa5f88c8d24478eb_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949282373/Put/seqid=0 2024-12-11T20:34:42,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T20:34:42,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741953_1129 (size=9714) 2024-12-11T20:34:42,481 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:42,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49746 deadline: 1733949342462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733949282400, max timestamp=1733949282482Num uniques 859; 2024-12-11T20:34:42,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-11T20:34:42,490 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733949282400, max timestamp=1733949282482Num uniques 859; 2024-12-11T20:34:42,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-11T20:34:42,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733949282400, max timestamp=1733949282482Num uniques 859; 2024-12-11T20:34:42,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-11T20:34:42,495 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211b998f90688f94e98aa5f88c8d24478eb_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b998f90688f94e98aa5f88c8d24478eb_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7fa9982e4ab5451580ebfa5736044023, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:42,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7fa9982e4ab5451580ebfa5736044023 is 175, key is test_row_0/A:col10/1733949282373/Put/seqid=0 2024-12-11T20:34:42,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:42,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:42,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741954_1130 (size=22361) 2024-12-11T20:34:42,544 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7fa9982e4ab5451580ebfa5736044023 2024-12-11T20:34:42,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49720 deadline: 1733949342540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949342552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949342556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949342556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/0352db70221c49bc9f6e09a3830569cd is 50, key is test_row_0/B:col10/1733949282373/Put/seqid=0 2024-12-11T20:34:42,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49746 deadline: 1733949342598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741955_1131 (size=9657) 2024-12-11T20:34:42,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/0352db70221c49bc9f6e09a3830569cd 2024-12-11T20:34:42,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49720 deadline: 1733949342667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T20:34:42,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949342678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,686 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949342678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949342682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/fd0bbe52c5be4a2cba4b7e9dfcea426e is 50, key is test_row_0/C:col10/1733949282373/Put/seqid=0 2024-12-11T20:34:42,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741956_1132 (size=9657) 2024-12-11T20:34:42,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/fd0bbe52c5be4a2cba4b7e9dfcea426e 2024-12-11T20:34:42,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7fa9982e4ab5451580ebfa5736044023 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7fa9982e4ab5451580ebfa5736044023 2024-12-11T20:34:42,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7fa9982e4ab5451580ebfa5736044023, entries=100, sequenceid=16, filesize=21.8 K 2024-12-11T20:34:42,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/0352db70221c49bc9f6e09a3830569cd as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/0352db70221c49bc9f6e09a3830569cd 2024-12-11T20:34:42,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/0352db70221c49bc9f6e09a3830569cd, entries=100, sequenceid=16, filesize=9.4 K 2024-12-11T20:34:42,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/fd0bbe52c5be4a2cba4b7e9dfcea426e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/fd0bbe52c5be4a2cba4b7e9dfcea426e 2024-12-11T20:34:42,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/fd0bbe52c5be4a2cba4b7e9dfcea426e, entries=100, sequenceid=16, filesize=9.4 K 2024-12-11T20:34:42,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 786cab87062764a85ac24e1e3653be4a in 390ms, sequenceid=16, compaction requested=false 2024-12-11T20:34:42,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:42,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:42,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=134.18 KB heapSize=370.92 KB 2024-12-11T20:34:42,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:42,824 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:42,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:42,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:42,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:42,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:42,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:42,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211976aabe8a7d246a09476559d256e516d_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949282820/Put/seqid=0 2024-12-11T20:34:42,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741957_1133 (size=14594) 2024-12-11T20:34:42,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949342898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49720 deadline: 1733949342896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949342899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949342899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49746 deadline: 1733949342914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:42,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:42,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:42,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:42,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:42,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T20:34:43,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49720 deadline: 1733949343027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49746 deadline: 1733949343027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,039 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:43,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:43,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949343227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949343227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949343227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49720 deadline: 1733949343247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49746 deadline: 1733949343248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:43,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:43,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,305 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:43,312 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211976aabe8a7d246a09476559d256e516d_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211976aabe8a7d246a09476559d256e516d_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:43,314 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/988bb673d2134f5491ed3832ce1ed893, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:43,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/988bb673d2134f5491ed3832ce1ed893 is 175, key is test_row_0/A:col10/1733949282820/Put/seqid=0 2024-12-11T20:34:43,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741958_1134 (size=39549) 2024-12-11T20:34:43,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:43,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:43,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T20:34:43,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49720 deadline: 1733949343567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49746 deadline: 1733949343567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:43,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949343737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949343747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949343748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:43,760 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/988bb673d2134f5491ed3832ce1ed893 2024-12-11T20:34:43,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:43,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:43,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/1f3d352946ab4731a2b9adc933b574f8 is 50, key is test_row_0/B:col10/1733949282820/Put/seqid=0 2024-12-11T20:34:43,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741959_1135 (size=12001) 2024-12-11T20:34:43,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:43,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:43,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:43,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:44,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:44,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49720 deadline: 1733949344077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:44,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49746 deadline: 1733949344077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:44,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:44,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:44,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:44,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:44,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:44,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:44,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:44,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:44,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:44,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:44,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/1f3d352946ab4731a2b9adc933b574f8 2024-12-11T20:34:44,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/e82fe2dd259c407b95bccb5cf6f7f288 is 50, key is test_row_0/C:col10/1733949282820/Put/seqid=0 2024-12-11T20:34:44,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741960_1136 (size=12001) 2024-12-11T20:34:44,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/e82fe2dd259c407b95bccb5cf6f7f288 2024-12-11T20:34:44,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/988bb673d2134f5491ed3832ce1ed893 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/988bb673d2134f5491ed3832ce1ed893 2024-12-11T20:34:44,298 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/988bb673d2134f5491ed3832ce1ed893, entries=200, sequenceid=50, filesize=38.6 K 2024-12-11T20:34:44,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/1f3d352946ab4731a2b9adc933b574f8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/1f3d352946ab4731a2b9adc933b574f8 2024-12-11T20:34:44,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/1f3d352946ab4731a2b9adc933b574f8, entries=150, sequenceid=50, filesize=11.7 K 2024-12-11T20:34:44,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/e82fe2dd259c407b95bccb5cf6f7f288 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/e82fe2dd259c407b95bccb5cf6f7f288 2024-12-11T20:34:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/e82fe2dd259c407b95bccb5cf6f7f288, entries=150, sequenceid=50, filesize=11.7 K 2024-12-11T20:34:44,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~370.88 KB/379776, currentSize=53.67 KB/54960 for 786cab87062764a85ac24e1e3653be4a in 1496ms, sequenceid=50, compaction requested=false 2024-12-11T20:34:44,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-11T20:34:44,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:44,400 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T20:34:44,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:44,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:44,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:44,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:44,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211443b9e7946ba475883c6306f2686e424_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949282911/Put/seqid=0 2024-12-11T20:34:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741961_1137 (size=9714) 2024-12-11T20:34:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,499 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211443b9e7946ba475883c6306f2686e424_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211443b9e7946ba475883c6306f2686e424_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/2a0745668bce4e0c8295c84200613e7f, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/2a0745668bce4e0c8295c84200613e7f is 175, key is test_row_0/A:col10/1733949282911/Put/seqid=0 2024-12-11T20:34:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T20:34:44,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741962_1138 (size=22361) 2024-12-11T20:34:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,539 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/2a0745668bce4e0c8295c84200613e7f 2024-12-11T20:34:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/ac48707e4c1048ccb67a20bbf8123f65 is 50, key is test_row_0/B:col10/1733949282911/Put/seqid=0 2024-12-11T20:34:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741963_1139 (size=9657) 2024-12-11T20:34:44,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,750 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:34:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:44,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949284858, max timestamp=1733949284884Num uniques 859; 2024-12-11T20:34:44,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949284858, max timestamp=1733949284885Num uniques 859; 2024-12-11T20:34:44,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:44,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:44,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949284858, max timestamp=1733949284885Num uniques 859; 2024-12-11T20:34:44,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:44,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:44,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949344944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:44,952 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:44,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:44,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949344948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:44,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949344950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:44,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:44,976 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/ac48707e4c1048ccb67a20bbf8123f65 2024-12-11T20:34:44,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/2a411e2c5f60487ea2a109d72e90f73b is 50, key is test_row_0/C:col10/1733949282911/Put/seqid=0 2024-12-11T20:34:45,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741964_1140 (size=9657) 2024-12-11T20:34:45,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949345057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949345067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949345076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49746 deadline: 1733949345097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49720 deadline: 1733949345097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,098 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_0' in 786cab87062764a85ac24e1e3653be4a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,099 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 786cab87062764a85ac24e1e3653be4a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949345267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949345276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949345287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,410 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/2a411e2c5f60487ea2a109d72e90f73b 2024-12-11T20:34:45,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/2a0745668bce4e0c8295c84200613e7f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a0745668bce4e0c8295c84200613e7f 2024-12-11T20:34:45,421 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a0745668bce4e0c8295c84200613e7f, entries=100, sequenceid=61, filesize=21.8 K 2024-12-11T20:34:45,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/ac48707e4c1048ccb67a20bbf8123f65 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/ac48707e4c1048ccb67a20bbf8123f65 2024-12-11T20:34:45,427 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/ac48707e4c1048ccb67a20bbf8123f65, entries=100, sequenceid=61, filesize=9.4 K 2024-12-11T20:34:45,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/2a411e2c5f60487ea2a109d72e90f73b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2a411e2c5f60487ea2a109d72e90f73b 2024-12-11T20:34:45,437 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2a411e2c5f60487ea2a109d72e90f73b, entries=100, sequenceid=61, filesize=9.4 K 2024-12-11T20:34:45,438 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 786cab87062764a85ac24e1e3653be4a in 1038ms, sequenceid=61, compaction requested=true 2024-12-11T20:34:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:45,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-11T20:34:45,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-11T20:34:45,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-11T20:34:45,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0750 sec 2024-12-11T20:34:45,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.0830 sec 2024-12-11T20:34:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:45,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-11T20:34:45,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:45,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:45,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:45,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:45,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:45,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:45,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412119888b401128b4bdb9c10093d673750cc_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949285577/Put/seqid=0 2024-12-11T20:34:45,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741965_1141 (size=14594) 2024-12-11T20:34:45,597 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:45,602 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412119888b401128b4bdb9c10093d673750cc_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412119888b401128b4bdb9c10093d673750cc_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:45,605 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/918e97d1756c4fb58643d6a9686b273d, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:45,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/918e97d1756c4fb58643d6a9686b273d is 175, key is test_row_0/A:col10/1733949285577/Put/seqid=0 2024-12-11T20:34:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949345606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949345607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949345608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741966_1142 (size=39549) 2024-12-11T20:34:45,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949345717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949345718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949345718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949345927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949345928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949345928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:45,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:46,034 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/918e97d1756c4fb58643d6a9686b273d 2024-12-11T20:34:46,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/02cfaddc47404fa7abaf39325843968b is 50, key is test_row_0/B:col10/1733949285577/Put/seqid=0 2024-12-11T20:34:46,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741967_1143 (size=12001) 2024-12-11T20:34:46,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949346237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:46,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949346237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:46,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949346237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:46,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/02cfaddc47404fa7abaf39325843968b 2024-12-11T20:34:46,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/3ffc61f9517d4accbc3ad391005dc711 is 50, key is test_row_0/C:col10/1733949285577/Put/seqid=0 2024-12-11T20:34:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-11T20:34:46,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741968_1144 (size=12001) 2024-12-11T20:34:46,528 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:46,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/3ffc61f9517d4accbc3ad391005dc711 2024-12-11T20:34:46,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-11T20:34:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-11T20:34:46,532 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:46,533 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:46,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:46,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/918e97d1756c4fb58643d6a9686b273d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/918e97d1756c4fb58643d6a9686b273d 2024-12-11T20:34:46,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/918e97d1756c4fb58643d6a9686b273d, entries=200, sequenceid=98, filesize=38.6 K 2024-12-11T20:34:46,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/02cfaddc47404fa7abaf39325843968b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/02cfaddc47404fa7abaf39325843968b 2024-12-11T20:34:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/02cfaddc47404fa7abaf39325843968b, entries=150, sequenceid=98, filesize=11.7 K 2024-12-11T20:34:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/3ffc61f9517d4accbc3ad391005dc711 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3ffc61f9517d4accbc3ad391005dc711 2024-12-11T20:34:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3ffc61f9517d4accbc3ad391005dc711, entries=150, sequenceid=98, filesize=11.7 K 2024-12-11T20:34:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 786cab87062764a85ac24e1e3653be4a in 985ms, sequenceid=98, compaction requested=true 2024-12-11T20:34:46,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:46,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:46,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:46,564 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:46,564 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,566 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:46,566 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:46,567 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/B is initiating minor compaction (all files) 2024-12-11T20:34:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:46,567 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/B in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,567 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/0352db70221c49bc9f6e09a3830569cd, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/1f3d352946ab4731a2b9adc933b574f8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/ac48707e4c1048ccb67a20bbf8123f65, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/02cfaddc47404fa7abaf39325843968b] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=42.3 K 2024-12-11T20:34:46,567 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 123820 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:46,567 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/A is initiating minor compaction (all files) 2024-12-11T20:34:46,567 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/A in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:46,567 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 0352db70221c49bc9f6e09a3830569cd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733949282373 2024-12-11T20:34:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,567 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7fa9982e4ab5451580ebfa5736044023, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/988bb673d2134f5491ed3832ce1ed893, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a0745668bce4e0c8295c84200613e7f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/918e97d1756c4fb58643d6a9686b273d] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=120.9 K 2024-12-11T20:34:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,568 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3d352946ab4731a2b9adc933b574f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733949282537 2024-12-11T20:34:46,568 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:46,568 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7fa9982e4ab5451580ebfa5736044023, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/988bb673d2134f5491ed3832ce1ed893, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a0745668bce4e0c8295c84200613e7f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/918e97d1756c4fb58643d6a9686b273d] 2024-12-11T20:34:46,568 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting ac48707e4c1048ccb67a20bbf8123f65, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733949282880 2024-12-11T20:34:46,568 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fa9982e4ab5451580ebfa5736044023, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733949282373 2024-12-11T20:34:46,568 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 02cfaddc47404fa7abaf39325843968b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733949284929 2024-12-11T20:34:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,569 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 988bb673d2134f5491ed3832ce1ed893, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733949282537 2024-12-11T20:34:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,569 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a0745668bce4e0c8295c84200613e7f, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733949282880 2024-12-11T20:34:46,570 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 918e97d1756c4fb58643d6a9686b273d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733949284929 2024-12-11T20:34:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,581 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:46,581 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#B#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,581 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/0c1f51ab3dc14e1c82f0f44463c1f403 is 50, key is test_row_0/B:col10/1733949285577/Put/seqid=0 2024-12-11T20:34:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,584 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211d3c606141045479a81e096dffaa2380b_786cab87062764a85ac24e1e3653be4a store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:46,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,587 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211d3c606141045479a81e096dffaa2380b_786cab87062764a85ac24e1e3653be4a, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:46,587 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d3c606141045479a81e096dffaa2380b_786cab87062764a85ac24e1e3653be4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741969_1145 (size=12139) 2024-12-11T20:34:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741970_1146 (size=4469) 2024-12-11T20:34:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-11T20:34:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,640 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#A#compaction#126 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:46,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,641 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/ff81c210480142e8a14f985d043fbab5 is 175, key is test_row_0/A:col10/1733949285577/Put/seqid=0 2024-12-11T20:34:46,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741971_1147 (size=31093) 2024-12-11T20:34:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,679 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/ff81c210480142e8a14f985d043fbab5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ff81c210480142e8a14f985d043fbab5 2024-12-11T20:34:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-11T20:34:46,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:46,687 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:34:46,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:46,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:46,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:46,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:46,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:46,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,692 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/A of 786cab87062764a85ac24e1e3653be4a into ff81c210480142e8a14f985d043fbab5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:46,692 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:46,692 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/A, priority=12, startTime=1733949286564; duration=0sec 2024-12-11T20:34:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,698 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,698 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:A 2024-12-11T20:34:46,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,698 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,701 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43316 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:46,701 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/C is initiating minor compaction (all files) 2024-12-11T20:34:46,701 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/C in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,701 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/fd0bbe52c5be4a2cba4b7e9dfcea426e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/e82fe2dd259c407b95bccb5cf6f7f288, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2a411e2c5f60487ea2a109d72e90f73b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3ffc61f9517d4accbc3ad391005dc711] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=42.3 K 2024-12-11T20:34:46,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121131b8e395da774d848b5a7f677b7dc0b9_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949285599/Put/seqid=0 2024-12-11T20:34:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,702 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd0bbe52c5be4a2cba4b7e9dfcea426e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733949282373 2024-12-11T20:34:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,703 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting e82fe2dd259c407b95bccb5cf6f7f288, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733949282537 2024-12-11T20:34:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,704 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a411e2c5f60487ea2a109d72e90f73b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733949282880 2024-12-11T20:34:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,705 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ffc61f9517d4accbc3ad391005dc711, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733949284929 2024-12-11T20:34:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,717 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#C#compaction#128 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:46,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,718 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/984a4796099a458eb9810e896fb8b040 is 50, key is test_row_0/C:col10/1733949285577/Put/seqid=0 2024-12-11T20:34:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741973_1149 (size=12139) 2024-12-11T20:34:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,748 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/984a4796099a458eb9810e896fb8b040 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/984a4796099a458eb9810e896fb8b040 2024-12-11T20:34:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741972_1148 (size=9714) 2024-12-11T20:34:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,759 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121131b8e395da774d848b5a7f677b7dc0b9_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121131b8e395da774d848b5a7f677b7dc0b9_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/2a1edcf5aabe48c18fdcf96679023022, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/2a1edcf5aabe48c18fdcf96679023022 is 175, key is test_row_0/A:col10/1733949285599/Put/seqid=0 2024-12-11T20:34:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,765 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/C of 786cab87062764a85ac24e1e3653be4a into 984a4796099a458eb9810e896fb8b040(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:46,765 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:46,765 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/C, priority=12, startTime=1733949286566; duration=0sec 2024-12-11T20:34:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,766 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:46,766 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:C 2024-12-11T20:34:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:46,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:46,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741974_1150 (size=22361) 2024-12-11T20:34:46,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,784 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=108, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/2a1edcf5aabe48c18fdcf96679023022 2024-12-11T20:34:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:46,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/d715ca0f00a0427cad320d9eddbf685b is 50, key is test_row_0/B:col10/1733949285599/Put/seqid=0 2024-12-11T20:34:46,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733949286807, max timestamp=1733949286821Num uniques 859; 2024-12-11T20:34:46,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T20:34:46,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949286807, max timestamp=1733949286821Num uniques 859; 2024-12-11T20:34:46,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:46,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733949286807, max timestamp=1733949286821Num uniques 859; 2024-12-11T20:34:46,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T20:34:46,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741975_1151 (size=9657) 2024-12-11T20:34:46,841 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/d715ca0f00a0427cad320d9eddbf685b 2024-12-11T20:34:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-11T20:34:46,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/7312334def524ed3a5cc191903d1c2b2 is 50, key is test_row_0/C:col10/1733949285599/Put/seqid=0 2024-12-11T20:34:46,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741976_1152 (size=9657) 2024-12-11T20:34:46,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:46,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949346869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:46,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:46,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949346869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:46,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949346871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:46,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949346977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:46,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949346977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:46,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949346977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:46,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,039 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/0c1f51ab3dc14e1c82f0f44463c1f403 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/0c1f51ab3dc14e1c82f0f44463c1f403 2024-12-11T20:34:47,047 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/B of 786cab87062764a85ac24e1e3653be4a into 0c1f51ab3dc14e1c82f0f44463c1f403(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:47,048 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:47,048 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/B, priority=12, startTime=1733949286564; duration=0sec 2024-12-11T20:34:47,048 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:47,048 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:B 2024-12-11T20:34:47,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49746 deadline: 1733949347107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,109 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_0' in 786cab87062764a85ac24e1e3653be4a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:47,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,109 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49720 deadline: 1733949347127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,129 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 786cab87062764a85ac24e1e3653be4a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-11T20:34:47,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949347187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949347187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949347187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,266 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/7312334def524ed3a5cc191903d1c2b2 2024-12-11T20:34:47,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/2a1edcf5aabe48c18fdcf96679023022 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a1edcf5aabe48c18fdcf96679023022 2024-12-11T20:34:47,281 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a1edcf5aabe48c18fdcf96679023022, entries=100, sequenceid=108, filesize=21.8 K 2024-12-11T20:34:47,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/d715ca0f00a0427cad320d9eddbf685b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/d715ca0f00a0427cad320d9eddbf685b 2024-12-11T20:34:47,288 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/d715ca0f00a0427cad320d9eddbf685b, entries=100, sequenceid=108, filesize=9.4 K 2024-12-11T20:34:47,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/7312334def524ed3a5cc191903d1c2b2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7312334def524ed3a5cc191903d1c2b2 2024-12-11T20:34:47,298 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7312334def524ed3a5cc191903d1c2b2, entries=100, sequenceid=108, filesize=9.4 K 2024-12-11T20:34:47,300 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for 786cab87062764a85ac24e1e3653be4a in 613ms, sequenceid=108, compaction requested=false 2024-12-11T20:34:47,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:47,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:47,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-11T20:34:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-11T20:34:47,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-11T20:34:47,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 769 msec 2024-12-11T20:34:47,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 777 msec 2024-12-11T20:34:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:47,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-12-11T20:34:47,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:47,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:47,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:47,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:47,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:47,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:47,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949347510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949347510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412111b97697cfb2b45db8da8df359ee7e360_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949286868/Put/seqid=0 2024-12-11T20:34:47,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,514 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949347510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741977_1153 (size=14794) 2024-12-11T20:34:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949347617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949347617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949347626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-11T20:34:47,667 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:47,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-11T20:34:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T20:34:47,673 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:47,678 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:47,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T20:34:47,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949347826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949347827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T20:34:47,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:47,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:47,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:47,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:47,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:47,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:47,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949347847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:47,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:47,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:47,949 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:47,953 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412111b97697cfb2b45db8da8df359ee7e360_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412111b97697cfb2b45db8da8df359ee7e360_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:47,954 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/18add2e22f054885a34e7ff1f93a8fe3, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:47,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/18add2e22f054885a34e7ff1f93a8fe3 is 175, key is test_row_0/A:col10/1733949286868/Put/seqid=0 2024-12-11T20:34:47,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741978_1154 (size=39749) 2024-12-11T20:34:47,960 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/18add2e22f054885a34e7ff1f93a8fe3 2024-12-11T20:34:47,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/57bc7405d2fd4f93aa46094851292876 is 50, key is test_row_0/B:col10/1733949286868/Put/seqid=0 2024-12-11T20:34:47,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741979_1155 (size=12151) 2024-12-11T20:34:47,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T20:34:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:47,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:47,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:47,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:47,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:47,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T20:34:48,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T20:34:48,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:48,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949348137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:48,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:48,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949348138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:48,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949348157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:48,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,162 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:48,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T20:34:48,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:48,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T20:34:48,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/57bc7405d2fd4f93aa46094851292876 2024-12-11T20:34:48,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/aef40cb60d5a41028b5dfdee5e3ab763 is 50, key is test_row_0/C:col10/1733949286868/Put/seqid=0 2024-12-11T20:34:48,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741980_1156 (size=12151) 2024-12-11T20:34:48,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T20:34:48,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:48,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T20:34:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:48,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949348647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:48,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949348647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:48,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949348677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:48,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:48,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T20:34:48,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:48,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:48,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/aef40cb60d5a41028b5dfdee5e3ab763 2024-12-11T20:34:48,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/18add2e22f054885a34e7ff1f93a8fe3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/18add2e22f054885a34e7ff1f93a8fe3 2024-12-11T20:34:48,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/18add2e22f054885a34e7ff1f93a8fe3, entries=200, sequenceid=148, filesize=38.8 K 2024-12-11T20:34:48,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/57bc7405d2fd4f93aa46094851292876 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/57bc7405d2fd4f93aa46094851292876 2024-12-11T20:34:48,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/57bc7405d2fd4f93aa46094851292876, entries=150, sequenceid=148, filesize=11.9 K 2024-12-11T20:34:48,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T20:34:48,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/aef40cb60d5a41028b5dfdee5e3ab763 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/aef40cb60d5a41028b5dfdee5e3ab763 2024-12-11T20:34:48,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/aef40cb60d5a41028b5dfdee5e3ab763, entries=150, sequenceid=148, filesize=11.9 K 2024-12-11T20:34:48,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=40.25 KB/41220 for 786cab87062764a85ac24e1e3653be4a in 1319ms, sequenceid=148, compaction requested=true 2024-12-11T20:34:48,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:48,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:48,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:48,819 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:48,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:48,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:48,819 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:48,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:48,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:48,826 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:48,820 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93203 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:48,826 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/B is initiating minor compaction (all files) 2024-12-11T20:34:48,826 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/A is initiating minor compaction (all files) 2024-12-11T20:34:48,826 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/B in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,826 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/A in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,827 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/0c1f51ab3dc14e1c82f0f44463c1f403, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/d715ca0f00a0427cad320d9eddbf685b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/57bc7405d2fd4f93aa46094851292876] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=33.2 K 2024-12-11T20:34:48,827 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ff81c210480142e8a14f985d043fbab5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a1edcf5aabe48c18fdcf96679023022, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/18add2e22f054885a34e7ff1f93a8fe3] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=91.0 K 2024-12-11T20:34:48,827 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,827 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ff81c210480142e8a14f985d043fbab5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a1edcf5aabe48c18fdcf96679023022, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/18add2e22f054885a34e7ff1f93a8fe3] 2024-12-11T20:34:48,827 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c1f51ab3dc14e1c82f0f44463c1f403, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733949284929 2024-12-11T20:34:48,827 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff81c210480142e8a14f985d043fbab5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733949284929 2024-12-11T20:34:48,828 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d715ca0f00a0427cad320d9eddbf685b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733949285599 2024-12-11T20:34:48,828 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a1edcf5aabe48c18fdcf96679023022, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733949285599 2024-12-11T20:34:48,828 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 57bc7405d2fd4f93aa46094851292876, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733949286863 2024-12-11T20:34:48,829 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18add2e22f054885a34e7ff1f93a8fe3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733949286856 2024-12-11T20:34:48,836 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#B#compaction#134 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:48,837 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/498dc3d191ea4cbba36ada571a10c664 is 50, key is test_row_0/B:col10/1733949286868/Put/seqid=0 2024-12-11T20:34:48,839 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:48,843 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211f013e71891654a6caaf40d65915ddad2_786cab87062764a85ac24e1e3653be4a store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:48,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741981_1157 (size=12391) 2024-12-11T20:34:48,845 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211f013e71891654a6caaf40d65915ddad2_786cab87062764a85ac24e1e3653be4a, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:48,846 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211f013e71891654a6caaf40d65915ddad2_786cab87062764a85ac24e1e3653be4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:48,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741982_1158 (size=4469) 2024-12-11T20:34:48,853 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#A#compaction#135 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:48,855 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/548d283de6ee4334a96ccb1e22c0e259 is 175, key is test_row_0/A:col10/1733949286868/Put/seqid=0 2024-12-11T20:34:48,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741983_1159 (size=31345) 2024-12-11T20:34:48,869 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/548d283de6ee4334a96ccb1e22c0e259 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/548d283de6ee4334a96ccb1e22c0e259 2024-12-11T20:34:48,879 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/A of 786cab87062764a85ac24e1e3653be4a into 548d283de6ee4334a96ccb1e22c0e259(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:48,879 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:48,879 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/A, priority=13, startTime=1733949288819; duration=0sec 2024-12-11T20:34:48,879 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:48,879 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:A 2024-12-11T20:34:48,879 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:48,881 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:48,881 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/C is initiating minor compaction (all files) 2024-12-11T20:34:48,881 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/C in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,881 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/984a4796099a458eb9810e896fb8b040, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7312334def524ed3a5cc191903d1c2b2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/aef40cb60d5a41028b5dfdee5e3ab763] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=33.2 K 2024-12-11T20:34:48,882 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 984a4796099a458eb9810e896fb8b040, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1733949284929 2024-12-11T20:34:48,883 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7312334def524ed3a5cc191903d1c2b2, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1733949285599 2024-12-11T20:34:48,884 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting aef40cb60d5a41028b5dfdee5e3ab763, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733949286863 2024-12-11T20:34:48,897 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#C#compaction#136 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:48,898 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/d589015e4afe4fb689e4bc6c0b60a3b3 is 50, key is test_row_0/C:col10/1733949286868/Put/seqid=0 2024-12-11T20:34:48,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741984_1160 (size=12391) 2024-12-11T20:34:48,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-11T20:34:48,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:48,919 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-11T20:34:48,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:48,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:48,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:48,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:48,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:48,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211195bd3328cd64c49a77f088ded0843a7_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949287508/Put/seqid=0 2024-12-11T20:34:48,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741985_1161 (size=12304) 2024-12-11T20:34:48,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:48,939 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211195bd3328cd64c49a77f088ded0843a7_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211195bd3328cd64c49a77f088ded0843a7_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:48,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/e65118c32d9540b796c591ed7aaf062f, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:48,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/e65118c32d9540b796c591ed7aaf062f is 175, key is test_row_0/A:col10/1733949287508/Put/seqid=0 2024-12-11T20:34:48,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741986_1162 (size=31105) 2024-12-11T20:34:49,251 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/498dc3d191ea4cbba36ada571a10c664 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/498dc3d191ea4cbba36ada571a10c664 2024-12-11T20:34:49,257 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/B of 786cab87062764a85ac24e1e3653be4a into 498dc3d191ea4cbba36ada571a10c664(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:49,257 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:49,257 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/B, priority=13, startTime=1733949288819; duration=0sec 2024-12-11T20:34:49,257 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:49,257 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:B 2024-12-11T20:34:49,314 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/d589015e4afe4fb689e4bc6c0b60a3b3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/d589015e4afe4fb689e4bc6c0b60a3b3 2024-12-11T20:34:49,321 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/C of 786cab87062764a85ac24e1e3653be4a into d589015e4afe4fb689e4bc6c0b60a3b3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:49,321 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:49,321 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/C, priority=13, startTime=1733949288819; duration=0sec 2024-12-11T20:34:49,321 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:49,321 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:C 2024-12-11T20:34:49,372 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/e65118c32d9540b796c591ed7aaf062f 2024-12-11T20:34:49,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/73eb0920bfd043e589bb6d6e6e1f3dbe is 50, key is test_row_0/B:col10/1733949287508/Put/seqid=0 2024-12-11T20:34:49,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741987_1163 (size=12151) 2024-12-11T20:34:49,425 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/73eb0920bfd043e589bb6d6e6e1f3dbe 2024-12-11T20:34:49,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/af14a1179e5a4aaaa632e48be85cbb00 is 50, key is test_row_0/C:col10/1733949287508/Put/seqid=0 2024-12-11T20:34:49,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741988_1164 (size=12151) 2024-12-11T20:34:49,478 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/af14a1179e5a4aaaa632e48be85cbb00 2024-12-11T20:34:49,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/e65118c32d9540b796c591ed7aaf062f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/e65118c32d9540b796c591ed7aaf062f 2024-12-11T20:34:49,492 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/e65118c32d9540b796c591ed7aaf062f, entries=150, sequenceid=158, filesize=30.4 K 2024-12-11T20:34:49,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/73eb0920bfd043e589bb6d6e6e1f3dbe as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73eb0920bfd043e589bb6d6e6e1f3dbe 2024-12-11T20:34:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,503 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73eb0920bfd043e589bb6d6e6e1f3dbe, entries=150, sequenceid=158, filesize=11.9 K 2024-12-11T20:34:49,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/af14a1179e5a4aaaa632e48be85cbb00 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af14a1179e5a4aaaa632e48be85cbb00 2024-12-11T20:34:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,515 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af14a1179e5a4aaaa632e48be85cbb00, entries=150, sequenceid=158, filesize=11.9 K 2024-12-11T20:34:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,516 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 786cab87062764a85ac24e1e3653be4a in 597ms, sequenceid=158, compaction requested=false 2024-12-11T20:34:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:49,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-11T20:34:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-11T20:34:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-11T20:34:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8400 sec 2024-12-11T20:34:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8540 sec 2024-12-11T20:34:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:34:49,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:49,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:49,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:49,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:49,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:49,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121147e0b445daa1497abbfdd304fc2b0437_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949289753/Put/seqid=0 2024-12-11T20:34:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741989_1165 (size=22268) 2024-12-11T20:34:49,808 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:49,817 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121147e0b445daa1497abbfdd304fc2b0437_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121147e0b445daa1497abbfdd304fc2b0437_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-11T20:34:49,818 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:49,818 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7c08166792ca4cb28d772c041a762c4d, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:49,818 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:49,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7c08166792ca4cb28d772c041a762c4d is 175, key is test_row_0/A:col10/1733949289753/Put/seqid=0 2024-12-11T20:34:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-11T20:34:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T20:34:49,822 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:49,823 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:49,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:49,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733949289809, max timestamp=1733949289854Num uniques 859; 2024-12-11T20:34:49,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-11T20:34:49,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733949289809, max timestamp=1733949289854Num uniques 859; 2024-12-11T20:34:49,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-11T20:34:49,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733949289809, max timestamp=1733949289854Num uniques 859; 2024-12-11T20:34:49,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-11T20:34:49,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741990_1166 (size=65669) 2024-12-11T20:34:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T20:34:49,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949349925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:49,934 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:49,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949349931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:49,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949349933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:49,936 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:49,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T20:34:49,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:49,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:49,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:49,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:49,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949350037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:50,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:50,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949350047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949350047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:50,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T20:34:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T20:34:50,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:50,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949350258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:50,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949350259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:50,262 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7c08166792ca4cb28d772c041a762c4d 2024-12-11T20:34:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:50,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949350259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:50,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T20:34:50,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:50,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/2a1752087e06430f8393ef9e7eda2cbf is 50, key is test_row_0/B:col10/1733949289753/Put/seqid=0 2024-12-11T20:34:50,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741991_1167 (size=9757) 2024-12-11T20:34:50,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T20:34:50,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:50,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T20:34:50,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949350566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:50,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:50,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:50,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949350566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:50,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949350567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:50,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:50,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T20:34:50,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:50,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/2a1752087e06430f8393ef9e7eda2cbf 2024-12-11T20:34:50,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/f256c897db8947cc9c6a19853c765efb is 50, key is test_row_0/C:col10/1733949289753/Put/seqid=0 2024-12-11T20:34:50,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741992_1168 (size=9757) 2024-12-11T20:34:50,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/f256c897db8947cc9c6a19853c765efb 2024-12-11T20:34:50,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7c08166792ca4cb28d772c041a762c4d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7c08166792ca4cb28d772c041a762c4d 2024-12-11T20:34:50,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7c08166792ca4cb28d772c041a762c4d, entries=350, sequenceid=170, filesize=64.1 K 2024-12-11T20:34:50,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/2a1752087e06430f8393ef9e7eda2cbf as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2a1752087e06430f8393ef9e7eda2cbf 2024-12-11T20:34:50,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2a1752087e06430f8393ef9e7eda2cbf, entries=100, sequenceid=170, filesize=9.5 K 2024-12-11T20:34:50,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/f256c897db8947cc9c6a19853c765efb as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/f256c897db8947cc9c6a19853c765efb 2024-12-11T20:34:50,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T20:34:50,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:50,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:50,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/f256c897db8947cc9c6a19853c765efb, entries=100, sequenceid=170, filesize=9.5 K 2024-12-11T20:34:50,756 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 786cab87062764a85ac24e1e3653be4a in 1001ms, sequenceid=170, compaction requested=true 2024-12-11T20:34:50,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:50,756 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:50,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:50,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:50,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:50,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:50,756 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:50,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:50,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:50,757 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128119 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:50,757 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/A is initiating minor compaction (all files) 2024-12-11T20:34:50,757 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/A in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,757 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/548d283de6ee4334a96ccb1e22c0e259, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/e65118c32d9540b796c591ed7aaf062f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7c08166792ca4cb28d772c041a762c4d] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=125.1 K 2024-12-11T20:34:50,757 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,758 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/548d283de6ee4334a96ccb1e22c0e259, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/e65118c32d9540b796c591ed7aaf062f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7c08166792ca4cb28d772c041a762c4d] 2024-12-11T20:34:50,758 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:50,758 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/B is initiating minor compaction (all files) 2024-12-11T20:34:50,758 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/B in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,758 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/498dc3d191ea4cbba36ada571a10c664, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73eb0920bfd043e589bb6d6e6e1f3dbe, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2a1752087e06430f8393ef9e7eda2cbf] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=33.5 K 2024-12-11T20:34:50,758 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 548d283de6ee4334a96ccb1e22c0e259, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733949286863 2024-12-11T20:34:50,759 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting e65118c32d9540b796c591ed7aaf062f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733949287501 2024-12-11T20:34:50,759 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c08166792ca4cb28d772c041a762c4d, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733949289682 2024-12-11T20:34:50,759 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 498dc3d191ea4cbba36ada571a10c664, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733949286863 2024-12-11T20:34:50,762 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 73eb0920bfd043e589bb6d6e6e1f3dbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733949287501 2024-12-11T20:34:50,763 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a1752087e06430f8393ef9e7eda2cbf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733949289752 2024-12-11T20:34:50,775 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:50,778 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:50,778 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/8a80a90be8bf49c18df7a4a25fa88b26 is 50, key is test_row_0/B:col10/1733949289753/Put/seqid=0 2024-12-11T20:34:50,787 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211bb3913f934cb4439b8e0e28d6d59dffa_786cab87062764a85ac24e1e3653be4a store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:50,790 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211bb3913f934cb4439b8e0e28d6d59dffa_786cab87062764a85ac24e1e3653be4a, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:50,790 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bb3913f934cb4439b8e0e28d6d59dffa_786cab87062764a85ac24e1e3653be4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:50,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741993_1169 (size=12493) 2024-12-11T20:34:50,816 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/8a80a90be8bf49c18df7a4a25fa88b26 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/8a80a90be8bf49c18df7a4a25fa88b26 2024-12-11T20:34:50,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741994_1170 (size=4469) 2024-12-11T20:34:50,823 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/B of 786cab87062764a85ac24e1e3653be4a into 8a80a90be8bf49c18df7a4a25fa88b26(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:50,823 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:50,823 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/B, priority=13, startTime=1733949290756; duration=0sec 2024-12-11T20:34:50,823 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:50,824 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:B 2024-12-11T20:34:50,824 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:50,825 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:50,826 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/C is initiating minor compaction (all files) 2024-12-11T20:34:50,826 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/C in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,826 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/d589015e4afe4fb689e4bc6c0b60a3b3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af14a1179e5a4aaaa632e48be85cbb00, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/f256c897db8947cc9c6a19853c765efb] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=33.5 K 2024-12-11T20:34:50,826 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d589015e4afe4fb689e4bc6c0b60a3b3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1733949286863 2024-12-11T20:34:50,826 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting af14a1179e5a4aaaa632e48be85cbb00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1733949287501 2024-12-11T20:34:50,827 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f256c897db8947cc9c6a19853c765efb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733949289752 2024-12-11T20:34:50,837 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#C#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:50,837 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/40f8fec814fe4ff38de027ef27c81303 is 50, key is test_row_0/C:col10/1733949289753/Put/seqid=0 2024-12-11T20:34:50,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741995_1171 (size=12493) 2024-12-11T20:34:50,863 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/40f8fec814fe4ff38de027ef27c81303 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/40f8fec814fe4ff38de027ef27c81303 2024-12-11T20:34:50,872 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/C of 786cab87062764a85ac24e1e3653be4a into 40f8fec814fe4ff38de027ef27c81303(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:50,873 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:50,873 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/C, priority=13, startTime=1733949290756; duration=0sec 2024-12-11T20:34:50,873 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:50,873 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:C 2024-12-11T20:34:50,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-11T20:34:50,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:50,903 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-11T20:34:50,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:50,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:50,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:50,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:50,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:50,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:50,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211f851e6dd0ed74791afa03a16f2ef44a4_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949289905/Put/seqid=0 2024-12-11T20:34:50,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741996_1172 (size=12304) 2024-12-11T20:34:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:50,933 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211f851e6dd0ed74791afa03a16f2ef44a4_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f851e6dd0ed74791afa03a16f2ef44a4_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:50,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/024c81c211fa4a6facb9b3d172a0959d, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:50,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/024c81c211fa4a6facb9b3d172a0959d is 175, key is test_row_0/A:col10/1733949289905/Put/seqid=0 2024-12-11T20:34:50,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741997_1173 (size=31105) 2024-12-11T20:34:50,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T20:34:51,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:51,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949351099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949351100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949351100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49746 deadline: 1733949351147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,152 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_0' in 786cab87062764a85ac24e1e3653be4a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49720 deadline: 1733949351167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,168 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 786cab87062764a85ac24e1e3653be4a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:34:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,169 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949351207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949351207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949351207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,219 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#A#compaction#143 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:51,219 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/273466f393de49808e4e0f0d68e8c638 is 175, key is test_row_0/A:col10/1733949289753/Put/seqid=0 2024-12-11T20:34:51,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741998_1174 (size=31554) 2024-12-11T20:34:51,356 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/024c81c211fa4a6facb9b3d172a0959d 2024-12-11T20:34:51,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/31ec2ec8f3ee40bd94606cad344f3786 is 50, key is test_row_0/B:col10/1733949289905/Put/seqid=0 2024-12-11T20:34:51,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741999_1175 (size=12151) 2024-12-11T20:34:51,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949351417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949351417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949351417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,632 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/273466f393de49808e4e0f0d68e8c638 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/273466f393de49808e4e0f0d68e8c638 2024-12-11T20:34:51,639 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/A of 786cab87062764a85ac24e1e3653be4a into 273466f393de49808e4e0f0d68e8c638(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:51,639 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:51,639 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/A, priority=13, startTime=1733949290756; duration=0sec 2024-12-11T20:34:51,639 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:51,639 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:A 2024-12-11T20:34:51,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949351727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949351727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949351727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:51,767 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/31ec2ec8f3ee40bd94606cad344f3786 2024-12-11T20:34:51,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/c5d42630b7684f0ca4e5b837f74b9c30 is 50, key is test_row_0/C:col10/1733949289905/Put/seqid=0 2024-12-11T20:34:51,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742000_1176 (size=12151) 2024-12-11T20:34:51,786 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/c5d42630b7684f0ca4e5b837f74b9c30 2024-12-11T20:34:51,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/024c81c211fa4a6facb9b3d172a0959d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/024c81c211fa4a6facb9b3d172a0959d 2024-12-11T20:34:51,801 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/024c81c211fa4a6facb9b3d172a0959d, entries=150, sequenceid=207, filesize=30.4 K 2024-12-11T20:34:51,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/31ec2ec8f3ee40bd94606cad344f3786 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/31ec2ec8f3ee40bd94606cad344f3786 2024-12-11T20:34:51,807 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/31ec2ec8f3ee40bd94606cad344f3786, entries=150, sequenceid=207, filesize=11.9 K 2024-12-11T20:34:51,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/c5d42630b7684f0ca4e5b837f74b9c30 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/c5d42630b7684f0ca4e5b837f74b9c30 2024-12-11T20:34:51,815 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/c5d42630b7684f0ca4e5b837f74b9c30, entries=150, sequenceid=207, filesize=11.9 K 2024-12-11T20:34:51,816 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=60.38 KB/61830 for 786cab87062764a85ac24e1e3653be4a in 913ms, sequenceid=207, compaction requested=false 2024-12-11T20:34:51,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:51,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:51,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-11T20:34:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-11T20:34:51,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-11T20:34:51,819 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9950 sec 2024-12-11T20:34:51,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.0020 sec 2024-12-11T20:34:51,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-11T20:34:51,968 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:51,969 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:51,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-11T20:34:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T20:34:51,974 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:51,975 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:51,975 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:52,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T20:34:52,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-11T20:34:52,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:52,130 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T20:34:52,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:52,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:52,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:52,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:52,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:52,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:52,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211549a5a75f4df4edea96333a36d63d969_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949291097/Put/seqid=0 2024-12-11T20:34:52,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742001_1177 (size=12304) 2024-12-11T20:34:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:52,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T20:34:52,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949352309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949352310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949352311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:52,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949352427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949352427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949352427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:52,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:52,583 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211549a5a75f4df4edea96333a36d63d969_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211549a5a75f4df4edea96333a36d63d969_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:52,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/ca35f0db55564127b719ef64c5fb3f1c, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:52,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/ca35f0db55564127b719ef64c5fb3f1c is 175, key is test_row_0/A:col10/1733949291097/Put/seqid=0 2024-12-11T20:34:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T20:34:52,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742002_1178 (size=31105) 2024-12-11T20:34:52,611 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/ca35f0db55564127b719ef64c5fb3f1c 2024-12-11T20:34:52,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/abdfb84856304d3b82828361d2567af6 is 50, key is test_row_0/B:col10/1733949291097/Put/seqid=0 2024-12-11T20:34:52,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949352637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:52,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949352637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949352637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:52,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742003_1179 (size=12151) 2024-12-11T20:34:52,651 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/abdfb84856304d3b82828361d2567af6 2024-12-11T20:34:52,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/2392396b8c0847e1b0934e03b7f7df2a is 50, key is test_row_0/C:col10/1733949291097/Put/seqid=0 2024-12-11T20:34:52,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742004_1180 (size=12151) 2024-12-11T20:34:52,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949352947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949352947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949352949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:52,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:53,096 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/2392396b8c0847e1b0934e03b7f7df2a 2024-12-11T20:34:53,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/ca35f0db55564127b719ef64c5fb3f1c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ca35f0db55564127b719ef64c5fb3f1c 2024-12-11T20:34:53,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T20:34:53,120 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ca35f0db55564127b719ef64c5fb3f1c, entries=150, sequenceid=220, filesize=30.4 K 2024-12-11T20:34:53,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/abdfb84856304d3b82828361d2567af6 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/abdfb84856304d3b82828361d2567af6 2024-12-11T20:34:53,132 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/abdfb84856304d3b82828361d2567af6, entries=150, sequenceid=220, filesize=11.9 K 2024-12-11T20:34:53,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/2392396b8c0847e1b0934e03b7f7df2a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2392396b8c0847e1b0934e03b7f7df2a 2024-12-11T20:34:53,148 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2392396b8c0847e1b0934e03b7f7df2a, entries=150, sequenceid=220, filesize=11.9 K 2024-12-11T20:34:53,150 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 786cab87062764a85ac24e1e3653be4a in 1019ms, sequenceid=220, compaction requested=true 2024-12-11T20:34:53,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:53,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:53,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-11T20:34:53,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-11T20:34:53,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-11T20:34:53,153 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1760 sec 2024-12-11T20:34:53,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.1850 sec 2024-12-11T20:34:53,189 INFO [master/5be53b084ac7:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-11T20:34:53,190 INFO [master/5be53b084ac7:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-11T20:34:53,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:53,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-11T20:34:53,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733949292306, max timestamp=1733949293459Num uniques 859; 2024-12-11T20:34:53,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-11T20:34:53,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733949292306, max timestamp=1733949293459Num uniques 859; 2024-12-11T20:34:53,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-11T20:34:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:53,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:53,472 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:53,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bfa2e5f231304751b159618b34a46ff1_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949293459/Put/seqid=0 2024-12-11T20:34:53,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742005_1181 (size=12304) 2024-12-11T20:34:53,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949353508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:53,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:53,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949353515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:53,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:53,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949353515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:53,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:53,525 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bfa2e5f231304751b159618b34a46ff1_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bfa2e5f231304751b159618b34a46ff1_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:53,527 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/63d29f522ef5480faac63eb6aaeb7eba, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:53,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/63d29f522ef5480faac63eb6aaeb7eba is 175, key is test_row_0/A:col10/1733949293459/Put/seqid=0 2024-12-11T20:34:53,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742006_1182 (size=31105) 2024-12-11T20:34:53,570 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/63d29f522ef5480faac63eb6aaeb7eba 2024-12-11T20:34:53,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/bab7b44613d648208c4db9d8d839e8b4 is 50, key is test_row_0/B:col10/1733949293459/Put/seqid=0 2024-12-11T20:34:53,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742007_1183 (size=12151) 2024-12-11T20:34:53,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.79 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/bab7b44613d648208c4db9d8d839e8b4 2024-12-11T20:34:53,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/ff084f09a2284ab58768ce58d45a8416 is 50, key is test_row_0/C:col10/1733949293459/Put/seqid=0 2024-12-11T20:34:53,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742008_1184 (size=12151) 2024-12-11T20:34:53,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949353617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:53,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.79 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/ff084f09a2284ab58768ce58d45a8416 2024-12-11T20:34:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:53,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949353627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:53,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:53,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949353627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:53,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/63d29f522ef5480faac63eb6aaeb7eba as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/63d29f522ef5480faac63eb6aaeb7eba 2024-12-11T20:34:53,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/63d29f522ef5480faac63eb6aaeb7eba, entries=150, sequenceid=242, filesize=30.4 K 2024-12-11T20:34:53,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/bab7b44613d648208c4db9d8d839e8b4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bab7b44613d648208c4db9d8d839e8b4 2024-12-11T20:34:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:53,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bab7b44613d648208c4db9d8d839e8b4, entries=150, sequenceid=242, filesize=11.9 K 2024-12-11T20:34:53,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/ff084f09a2284ab58768ce58d45a8416 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ff084f09a2284ab58768ce58d45a8416 2024-12-11T20:34:53,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ff084f09a2284ab58768ce58d45a8416, entries=150, sequenceid=242, filesize=11.9 K 2024-12-11T20:34:53,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.06 KB/65600, heapSize ~182.22 KB/186592, currentSize=114.05 KB/116790 for 786cab87062764a85ac24e1e3653be4a in 200ms, sequenceid=242, compaction requested=true 2024-12-11T20:34:53,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:53,662 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:53,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:53,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:53,662 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:53,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:53,665 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124869 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:53,665 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/A is initiating minor compaction (all files) 2024-12-11T20:34:53,666 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/A in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:53,666 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/273466f393de49808e4e0f0d68e8c638, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/024c81c211fa4a6facb9b3d172a0959d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ca35f0db55564127b719ef64c5fb3f1c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/63d29f522ef5480faac63eb6aaeb7eba] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=121.9 K 2024-12-11T20:34:53,666 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:53,666 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/273466f393de49808e4e0f0d68e8c638, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/024c81c211fa4a6facb9b3d172a0959d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ca35f0db55564127b719ef64c5fb3f1c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/63d29f522ef5480faac63eb6aaeb7eba] 2024-12-11T20:34:53,666 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:53,666 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/B is initiating minor compaction (all files) 2024-12-11T20:34:53,666 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/B in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:53,667 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/8a80a90be8bf49c18df7a4a25fa88b26, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/31ec2ec8f3ee40bd94606cad344f3786, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/abdfb84856304d3b82828361d2567af6, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bab7b44613d648208c4db9d8d839e8b4] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=47.8 K 2024-12-11T20:34:53,667 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 273466f393de49808e4e0f0d68e8c638, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733949287501 2024-12-11T20:34:53,667 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a80a90be8bf49c18df7a4a25fa88b26, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733949287501 2024-12-11T20:34:53,667 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 31ec2ec8f3ee40bd94606cad344f3786, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733949289893 2024-12-11T20:34:53,667 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 024c81c211fa4a6facb9b3d172a0959d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733949289893 2024-12-11T20:34:53,668 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting abdfb84856304d3b82828361d2567af6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733949291097 2024-12-11T20:34:53,668 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca35f0db55564127b719ef64c5fb3f1c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733949291097 2024-12-11T20:34:53,668 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63d29f522ef5480faac63eb6aaeb7eba, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733949292307 2024-12-11T20:34:53,668 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting bab7b44613d648208c4db9d8d839e8b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733949292307 2024-12-11T20:34:53,697 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#B#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:53,697 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/e6b68ef3b00f40048d71eeaccad96ad8 is 50, key is test_row_0/B:col10/1733949293459/Put/seqid=0 2024-12-11T20:34:53,699 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:53,704 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211d3662765c05841cd816788555b787cba_786cab87062764a85ac24e1e3653be4a store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:53,707 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211d3662765c05841cd816788555b787cba_786cab87062764a85ac24e1e3653be4a, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:53,707 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d3662765c05841cd816788555b787cba_786cab87062764a85ac24e1e3653be4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:53,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742009_1185 (size=12629) 2024-12-11T20:34:53,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742010_1186 (size=4469) 2024-12-11T20:34:53,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949293500, max timestamp=1733949293828Num uniques 859; 2024-12-11T20:34:53,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:34:53,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:53,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-11T20:34:53,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:53,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:53,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:53,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:53,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:53,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:34:53,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121167d3b110573f43329b1e6d523fc7b598_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949293828/Put/seqid=0 2024-12-11T20:34:53,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742011_1187 (size=14994) 2024-12-11T20:34:53,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:53,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:53,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949353872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:53,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121167d3b110573f43329b1e6d523fc7b598_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121167d3b110573f43329b1e6d523fc7b598_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:53,880 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/f6adf086074148bcafe168a22bc89347, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:53,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/f6adf086074148bcafe168a22bc89347 is 175, key is test_row_0/A:col10/1733949293828/Put/seqid=0 2024-12-11T20:34:53,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949353875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:53,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949353875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:53,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742012_1188 (size=39949) 2024-12-11T20:34:53,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949353987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:53,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:54,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949353998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:54,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949353998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:54,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-11T20:34:54,117 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:54,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-11T20:34:54,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T20:34:54,123 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:54,126 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:54,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:54,132 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/e6b68ef3b00f40048d71eeaccad96ad8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/e6b68ef3b00f40048d71eeaccad96ad8 2024-12-11T20:34:54,142 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/B of 786cab87062764a85ac24e1e3653be4a into e6b68ef3b00f40048d71eeaccad96ad8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:54,142 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:54,142 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/B, priority=12, startTime=1733949293662; duration=0sec 2024-12-11T20:34:54,142 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:54,142 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:B 2024-12-11T20:34:54,142 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:54,144 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:54,144 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/C is initiating minor compaction (all files) 2024-12-11T20:34:54,144 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/C in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:54,144 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/40f8fec814fe4ff38de027ef27c81303, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/c5d42630b7684f0ca4e5b837f74b9c30, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2392396b8c0847e1b0934e03b7f7df2a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ff084f09a2284ab58768ce58d45a8416] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=47.8 K 2024-12-11T20:34:54,146 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 40f8fec814fe4ff38de027ef27c81303, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1733949287501 2024-12-11T20:34:54,146 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c5d42630b7684f0ca4e5b837f74b9c30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1733949289893 2024-12-11T20:34:54,146 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 2392396b8c0847e1b0934e03b7f7df2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1733949291097 2024-12-11T20:34:54,147 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting ff084f09a2284ab58768ce58d45a8416, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733949292307 2024-12-11T20:34:54,160 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#A#compaction#156 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:54,161 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/8960bd7a47ec4ec8915814380255e62a is 175, key is test_row_0/A:col10/1733949293459/Put/seqid=0 2024-12-11T20:34:54,164 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#C#compaction#158 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:54,165 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/3a3620d93ebc4d158ab89ad1206a3321 is 50, key is test_row_0/C:col10/1733949293459/Put/seqid=0 2024-12-11T20:34:54,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949354197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:54,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742014_1190 (size=12629) 2024-12-11T20:34:54,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742013_1189 (size=31583) 2024-12-11T20:34:54,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949354208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949354208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:54,214 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/3a3620d93ebc4d158ab89ad1206a3321 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3a3620d93ebc4d158ab89ad1206a3321 2024-12-11T20:34:54,225 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/C of 786cab87062764a85ac24e1e3653be4a into 3a3620d93ebc4d158ab89ad1206a3321(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:54,225 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:54,225 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/C, priority=12, startTime=1733949293663; duration=0sec 2024-12-11T20:34:54,225 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:54,225 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:C 2024-12-11T20:34:54,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T20:34:54,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-11T20:34:54,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:54,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:54,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:54,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,329 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/f6adf086074148bcafe168a22bc89347 2024-12-11T20:34:54,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/5fae95f87dcc4c9b92a0596581cdb99d is 50, key is test_row_0/B:col10/1733949293828/Put/seqid=0 2024-12-11T20:34:54,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742015_1191 (size=12301) 2024-12-11T20:34:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T20:34:54,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-11T20:34:54,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:54,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:54,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:54,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:54,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949354507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:54,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949354517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:54,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:54,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949354517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:54,524 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:54,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-11T20:34:54,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:54,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:54,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:54,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,653 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/8960bd7a47ec4ec8915814380255e62a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/8960bd7a47ec4ec8915814380255e62a 2024-12-11T20:34:54,666 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/A of 786cab87062764a85ac24e1e3653be4a into 8960bd7a47ec4ec8915814380255e62a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-12-11T20:34:54,666 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:54,666 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/A, priority=12, startTime=1733949293662; duration=1sec 2024-12-11T20:34:54,666 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:54,666 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:A 2024-12-11T20:34:54,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-11T20:34:54,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:54,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:54,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:54,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T20:34:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:54,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/5fae95f87dcc4c9b92a0596581cdb99d 2024-12-11T20:34:54,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/8fe7f046b505440d8073867975097c87 is 50, key is test_row_0/C:col10/1733949293828/Put/seqid=0 2024-12-11T20:34:54,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742016_1192 (size=12301) 2024-12-11T20:34:54,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/8fe7f046b505440d8073867975097c87 2024-12-11T20:34:54,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/f6adf086074148bcafe168a22bc89347 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f6adf086074148bcafe168a22bc89347 2024-12-11T20:34:54,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f6adf086074148bcafe168a22bc89347, entries=200, sequenceid=263, filesize=39.0 K 2024-12-11T20:34:54,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/5fae95f87dcc4c9b92a0596581cdb99d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/5fae95f87dcc4c9b92a0596581cdb99d 2024-12-11T20:34:54,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/5fae95f87dcc4c9b92a0596581cdb99d, entries=150, sequenceid=263, filesize=12.0 K 2024-12-11T20:34:54,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/8fe7f046b505440d8073867975097c87 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/8fe7f046b505440d8073867975097c87 2024-12-11T20:34:54,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/8fe7f046b505440d8073867975097c87, entries=150, sequenceid=263, filesize=12.0 K 2024-12-11T20:34:54,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 786cab87062764a85ac24e1e3653be4a in 1022ms, sequenceid=263, compaction requested=false 2024-12-11T20:34:54,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:54,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-11T20:34:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:54,900 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:34:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:54,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:54,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e3d5f84d1843403d82eb4a9dea628960_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949293871/Put/seqid=0 2024-12-11T20:34:54,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742017_1193 (size=12454) 2024-12-11T20:34:54,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:54,958 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e3d5f84d1843403d82eb4a9dea628960_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e3d5f84d1843403d82eb4a9dea628960_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:54,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/863ada196d5249bfbce3e50726ca0245, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:54,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/863ada196d5249bfbce3e50726ca0245 is 175, key is test_row_0/A:col10/1733949293871/Put/seqid=0 2024-12-11T20:34:54,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742018_1194 (size=31255) 2024-12-11T20:34:54,974 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/863ada196d5249bfbce3e50726ca0245 2024-12-11T20:34:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/bb7fdf6ed3c548e999ddac71fcc198a7 is 50, key is test_row_0/B:col10/1733949293871/Put/seqid=0 2024-12-11T20:34:55,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742019_1195 (size=12301) 2024-12-11T20:34:55,007 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/bb7fdf6ed3c548e999ddac71fcc198a7 2024-12-11T20:34:55,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/af1f6a70c1094fc88922590c603a91f4 is 50, key is test_row_0/C:col10/1733949293871/Put/seqid=0 2024-12-11T20:34:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:55,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:55,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742020_1196 (size=12301) 2024-12-11T20:34:55,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949355069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949355070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949355087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949355187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949355187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949355197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,204 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T20:34:55,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949355397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949355397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949355407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,433 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/af1f6a70c1094fc88922590c603a91f4 2024-12-11T20:34:55,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/863ada196d5249bfbce3e50726ca0245 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/863ada196d5249bfbce3e50726ca0245 2024-12-11T20:34:55,448 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/863ada196d5249bfbce3e50726ca0245, entries=150, sequenceid=282, filesize=30.5 K 2024-12-11T20:34:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/bb7fdf6ed3c548e999ddac71fcc198a7 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bb7fdf6ed3c548e999ddac71fcc198a7 2024-12-11T20:34:55,458 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bb7fdf6ed3c548e999ddac71fcc198a7, entries=150, sequenceid=282, filesize=12.0 K 2024-12-11T20:34:55,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/af1f6a70c1094fc88922590c603a91f4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af1f6a70c1094fc88922590c603a91f4 2024-12-11T20:34:55,470 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af1f6a70c1094fc88922590c603a91f4, entries=150, sequenceid=282, filesize=12.0 K 2024-12-11T20:34:55,471 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 786cab87062764a85ac24e1e3653be4a in 571ms, sequenceid=282, compaction requested=true 2024-12-11T20:34:55,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:55,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:55,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-11T20:34:55,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-11T20:34:55,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-11T20:34:55,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3480 sec 2024-12-11T20:34:55,484 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.3590 sec 2024-12-11T20:34:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:55,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T20:34:55,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:55,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:55,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:55,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:55,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:55,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:55,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115465867a93dd4e4da5eb5a7c471bdf54_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949295077/Put/seqid=0 2024-12-11T20:34:55,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949355737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949355741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949355743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742021_1197 (size=17534) 2024-12-11T20:34:55,758 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:55,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115465867a93dd4e4da5eb5a7c471bdf54_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115465867a93dd4e4da5eb5a7c471bdf54_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:55,766 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7f2fe3e54f634e6b92ebc48c5088a8af, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:55,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7f2fe3e54f634e6b92ebc48c5088a8af is 175, key is test_row_0/A:col10/1733949295077/Put/seqid=0 2024-12-11T20:34:55,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742022_1198 (size=48639) 2024-12-11T20:34:55,774 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7f2fe3e54f634e6b92ebc48c5088a8af 2024-12-11T20:34:55,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/73ba9abb716840468769de497090effb is 50, key is test_row_0/B:col10/1733949295077/Put/seqid=0 2024-12-11T20:34:55,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742023_1199 (size=12301) 2024-12-11T20:34:55,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949355846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949355857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949355857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:56,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949356056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:56,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949356067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949356067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:56,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/73ba9abb716840468769de497090effb 2024-12-11T20:34:56,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/3fd406df92ba418eba7ebf0714a6f035 is 50, key is test_row_0/C:col10/1733949295077/Put/seqid=0 2024-12-11T20:34:56,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742024_1200 (size=12301) 2024-12-11T20:34:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-11T20:34:56,268 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:56,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-11T20:34:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-11T20:34:56,272 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:56,272 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:56,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:56,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949356367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:56,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-11T20:34:56,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949356377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:56,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949356378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:56,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-11T20:34:56,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:56,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:56,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:56,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:56,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:56,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:56,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-11T20:34:56,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:56,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:56,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:56,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:56,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-11T20:34:56,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/3fd406df92ba418eba7ebf0714a6f035 2024-12-11T20:34:56,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/7f2fe3e54f634e6b92ebc48c5088a8af as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7f2fe3e54f634e6b92ebc48c5088a8af 2024-12-11T20:34:56,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7f2fe3e54f634e6b92ebc48c5088a8af, entries=250, sequenceid=300, filesize=47.5 K 2024-12-11T20:34:56,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/73ba9abb716840468769de497090effb as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73ba9abb716840468769de497090effb 2024-12-11T20:34:56,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73ba9abb716840468769de497090effb, entries=150, sequenceid=300, filesize=12.0 K 2024-12-11T20:34:56,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/3fd406df92ba418eba7ebf0714a6f035 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3fd406df92ba418eba7ebf0714a6f035 2024-12-11T20:34:56,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3fd406df92ba418eba7ebf0714a6f035, entries=150, sequenceid=300, filesize=12.0 K 2024-12-11T20:34:56,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 786cab87062764a85ac24e1e3653be4a in 964ms, sequenceid=300, compaction requested=true 2024-12-11T20:34:56,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:56,680 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:34:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:56,681 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:34:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:56,682 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 151426 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:56,682 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/A is initiating minor compaction (all files) 2024-12-11T20:34:56,682 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/A in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:56,682 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/8960bd7a47ec4ec8915814380255e62a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f6adf086074148bcafe168a22bc89347, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/863ada196d5249bfbce3e50726ca0245, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7f2fe3e54f634e6b92ebc48c5088a8af] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=147.9 K 2024-12-11T20:34:56,682 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:56,682 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/8960bd7a47ec4ec8915814380255e62a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f6adf086074148bcafe168a22bc89347, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/863ada196d5249bfbce3e50726ca0245, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7f2fe3e54f634e6b92ebc48c5088a8af] 2024-12-11T20:34:56,683 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:34:56,683 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8960bd7a47ec4ec8915814380255e62a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733949292307 2024-12-11T20:34:56,684 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/B is initiating minor compaction (all files) 2024-12-11T20:34:56,684 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/B in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:56,684 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/e6b68ef3b00f40048d71eeaccad96ad8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/5fae95f87dcc4c9b92a0596581cdb99d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bb7fdf6ed3c548e999ddac71fcc198a7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73ba9abb716840468769de497090effb] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=48.4 K 2024-12-11T20:34:56,684 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6adf086074148bcafe168a22bc89347, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733949293500 2024-12-11T20:34:56,684 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b68ef3b00f40048d71eeaccad96ad8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733949292307 2024-12-11T20:34:56,684 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 863ada196d5249bfbce3e50726ca0245, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733949293861 2024-12-11T20:34:56,685 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fae95f87dcc4c9b92a0596581cdb99d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733949293506 2024-12-11T20:34:56,686 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting bb7fdf6ed3c548e999ddac71fcc198a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733949293861 2024-12-11T20:34:56,686 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f2fe3e54f634e6b92ebc48c5088a8af, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733949295050 2024-12-11T20:34:56,689 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 73ba9abb716840468769de497090effb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733949295077 2024-12-11T20:34:56,712 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:56,721 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:56,721 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/dd33786d07924e6fa2b7d5974ce41b8e is 50, key is test_row_0/B:col10/1733949295077/Put/seqid=0 2024-12-11T20:34:56,725 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412111956c65f6b254b04a212b2e1f4a9ec81_786cab87062764a85ac24e1e3653be4a store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:56,728 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412111956c65f6b254b04a212b2e1f4a9ec81_786cab87062764a85ac24e1e3653be4a, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:56,728 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412111956c65f6b254b04a212b2e1f4a9ec81_786cab87062764a85ac24e1e3653be4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:56,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-11T20:34:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:56,734 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-11T20:34:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:56,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:56,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742025_1201 (size=12915) 2024-12-11T20:34:56,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742026_1202 (size=4469) 2024-12-11T20:34:56,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121156cc32ed94314554922a6a40053d2117_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949295741/Put/seqid=0 2024-12-11T20:34:56,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742027_1203 (size=12454) 2024-12-11T20:34:56,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:56,829 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121156cc32ed94314554922a6a40053d2117_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121156cc32ed94314554922a6a40053d2117_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:56,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/c225980247d84d2e98fc567115550c13, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:56,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/c225980247d84d2e98fc567115550c13 is 175, key is test_row_0/A:col10/1733949295741/Put/seqid=0 2024-12-11T20:34:56,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742028_1204 (size=31255) 2024-12-11T20:34:56,845 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/c225980247d84d2e98fc567115550c13 2024-12-11T20:34:56,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/9d0986c4828a4e338d47c627b656418f is 50, key is test_row_0/B:col10/1733949295741/Put/seqid=0 2024-12-11T20:34:56,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:56,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:56,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742029_1205 (size=12301) 2024-12-11T20:34:56,887 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/9d0986c4828a4e338d47c627b656418f 2024-12-11T20:34:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-11T20:34:56,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/63902497595d4c8394814c830ca824d4 is 50, key is test_row_0/C:col10/1733949295741/Put/seqid=0 2024-12-11T20:34:56,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:56,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742030_1206 (size=12301) 2024-12-11T20:34:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949356915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:56,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949356915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:56,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:56,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949356916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:56,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:56,922 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/63902497595d4c8394814c830ca824d4 2024-12-11T20:34:56,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/c225980247d84d2e98fc567115550c13 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/c225980247d84d2e98fc567115550c13 2024-12-11T20:34:56,935 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/c225980247d84d2e98fc567115550c13, entries=150, sequenceid=314, filesize=30.5 K 2024-12-11T20:34:56,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/9d0986c4828a4e338d47c627b656418f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/9d0986c4828a4e338d47c627b656418f 2024-12-11T20:34:56,943 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/9d0986c4828a4e338d47c627b656418f, entries=150, sequenceid=314, filesize=12.0 K 2024-12-11T20:34:56,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/63902497595d4c8394814c830ca824d4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/63902497595d4c8394814c830ca824d4 2024-12-11T20:34:56,953 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/63902497595d4c8394814c830ca824d4, entries=150, sequenceid=314, filesize=12.0 K 2024-12-11T20:34:56,956 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 786cab87062764a85ac24e1e3653be4a in 221ms, sequenceid=314, compaction requested=true 2024-12-11T20:34:56,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:56,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:56,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-11T20:34:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-11T20:34:56,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-11T20:34:56,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 685 msec 2024-12-11T20:34:56,964 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 692 msec 2024-12-11T20:34:57,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-11T20:34:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:57,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:57,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121140f3f9bdb11b48bf8f7322ee3f31db54_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949296914/Put/seqid=0 2024-12-11T20:34:57,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949357053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949357054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949357061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742031_1207 (size=12454) 2024-12-11T20:34:57,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:57,109 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121140f3f9bdb11b48bf8f7322ee3f31db54_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121140f3f9bdb11b48bf8f7322ee3f31db54_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:57,111 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/a3de92896d4d458b8511f3590384bd49, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:57,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/a3de92896d4d458b8511f3590384bd49 is 175, key is test_row_0/A:col10/1733949296914/Put/seqid=0 2024-12-11T20:34:57,158 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/dd33786d07924e6fa2b7d5974ce41b8e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/dd33786d07924e6fa2b7d5974ce41b8e 2024-12-11T20:34:57,165 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/B of 786cab87062764a85ac24e1e3653be4a into dd33786d07924e6fa2b7d5974ce41b8e(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:57,165 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:57,165 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/B, priority=12, startTime=1733949296681; duration=0sec 2024-12-11T20:34:57,166 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:57,166 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:B 2024-12-11T20:34:57,166 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-11T20:34:57,167 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#A#compaction#167 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:57,168 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/0bebb45622c54783a20d0b5eae605eb1 is 175, key is test_row_0/A:col10/1733949295077/Put/seqid=0 2024-12-11T20:34:57,172 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61833 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-11T20:34:57,172 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/C is initiating minor compaction (all files) 2024-12-11T20:34:57,172 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/C in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:57,172 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3a3620d93ebc4d158ab89ad1206a3321, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/8fe7f046b505440d8073867975097c87, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af1f6a70c1094fc88922590c603a91f4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3fd406df92ba418eba7ebf0714a6f035, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/63902497595d4c8394814c830ca824d4] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=60.4 K 2024-12-11T20:34:57,173 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a3620d93ebc4d158ab89ad1206a3321, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733949292307 2024-12-11T20:34:57,174 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fe7f046b505440d8073867975097c87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1733949293506 2024-12-11T20:34:57,174 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting af1f6a70c1094fc88922590c603a91f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1733949293861 2024-12-11T20:34:57,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742032_1208 (size=31255) 2024-12-11T20:34:57,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949357167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,175 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fd406df92ba418eba7ebf0714a6f035, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733949295077 2024-12-11T20:34:57,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,175 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/a3de92896d4d458b8511f3590384bd49 2024-12-11T20:34:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949357167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949357167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,177 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 63902497595d4c8394814c830ca824d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733949295734 2024-12-11T20:34:57,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/2444f302d3c445348da5adad705d4f05 is 50, key is test_row_0/B:col10/1733949296914/Put/seqid=0 2024-12-11T20:34:57,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742033_1209 (size=31869) 2024-12-11T20:34:57,200 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:57,201 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/596c8bf3b6254b30a7c218d32ac49451 is 50, key is test_row_0/C:col10/1733949295741/Put/seqid=0 2024-12-11T20:34:57,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742034_1210 (size=12301) 2024-12-11T20:34:57,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/2444f302d3c445348da5adad705d4f05 2024-12-11T20:34:57,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742035_1211 (size=12949) 2024-12-11T20:34:57,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/861ae5b4deda454592bd3dd2e49aa510 is 50, key is test_row_0/C:col10/1733949296914/Put/seqid=0 2024-12-11T20:34:57,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742036_1212 (size=12301) 2024-12-11T20:34:57,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/861ae5b4deda454592bd3dd2e49aa510 2024-12-11T20:34:57,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/a3de92896d4d458b8511f3590384bd49 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/a3de92896d4d458b8511f3590384bd49 2024-12-11T20:34:57,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/a3de92896d4d458b8511f3590384bd49, entries=150, sequenceid=334, filesize=30.5 K 2024-12-11T20:34:57,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/2444f302d3c445348da5adad705d4f05 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2444f302d3c445348da5adad705d4f05 2024-12-11T20:34:57,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2444f302d3c445348da5adad705d4f05, entries=150, sequenceid=334, filesize=12.0 K 2024-12-11T20:34:57,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/861ae5b4deda454592bd3dd2e49aa510 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/861ae5b4deda454592bd3dd2e49aa510 2024-12-11T20:34:57,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/861ae5b4deda454592bd3dd2e49aa510, entries=150, sequenceid=334, filesize=12.0 K 2024-12-11T20:34:57,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 786cab87062764a85ac24e1e3653be4a in 251ms, sequenceid=334, compaction requested=true 2024-12-11T20:34:57,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:57,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:A, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:57,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:57,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:B, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:57,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T20:34:57,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:57,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-11T20:34:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:57,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T20:34:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:57,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:57,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412117f24028f330d47c5a3150b9a366f5503_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949297390/Put/seqid=0 2024-12-11T20:34:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-11T20:34:57,407 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:57,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-11T20:34:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-11T20:34:57,409 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:57,410 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:57,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742037_1213 (size=12454) 2024-12-11T20:34:57,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:57,413 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:57,423 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412117f24028f330d47c5a3150b9a366f5503_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117f24028f330d47c5a3150b9a366f5503_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:57,425 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/9dfdc5bd18ef4c168c9128146d6719db, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:57,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/9dfdc5bd18ef4c168c9128146d6719db is 175, key is test_row_0/A:col10/1733949297390/Put/seqid=0 2024-12-11T20:34:57,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742038_1214 (size=31255) 2024-12-11T20:34:57,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949357441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949357453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949357454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-11T20:34:57,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949357557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-11T20:34:57,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:57,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:57,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:57,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:57,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949357567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949357567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,605 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/0bebb45622c54783a20d0b5eae605eb1 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/0bebb45622c54783a20d0b5eae605eb1 2024-12-11T20:34:57,615 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/A of 786cab87062764a85ac24e1e3653be4a into 0bebb45622c54783a20d0b5eae605eb1(size=31.1 K), total size for store is 92.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:57,616 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:57,616 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/A, priority=12, startTime=1733949296680; duration=0sec 2024-12-11T20:34:57,616 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-11T20:34:57,616 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:A 2024-12-11T20:34:57,616 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:A 2024-12-11T20:34:57,616 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 5 compacting, 1 eligible, 16 blocking 2024-12-11T20:34:57,617 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-11T20:34:57,617 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-11T20:34:57,617 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. because compaction request was cancelled 2024-12-11T20:34:57,617 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:C 2024-12-11T20:34:57,617 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:57,619 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:57,619 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/A is initiating minor compaction (all files) 2024-12-11T20:34:57,619 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/A in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:57,619 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/0bebb45622c54783a20d0b5eae605eb1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/c225980247d84d2e98fc567115550c13, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/a3de92896d4d458b8511f3590384bd49] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=92.2 K 2024-12-11T20:34:57,619 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:57,619 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/0bebb45622c54783a20d0b5eae605eb1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/c225980247d84d2e98fc567115550c13, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/a3de92896d4d458b8511f3590384bd49] 2024-12-11T20:34:57,620 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bebb45622c54783a20d0b5eae605eb1, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733949295077 2024-12-11T20:34:57,620 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting c225980247d84d2e98fc567115550c13, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733949295734 2024-12-11T20:34:57,621 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3de92896d4d458b8511f3590384bd49, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733949296914 2024-12-11T20:34:57,632 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/596c8bf3b6254b30a7c218d32ac49451 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/596c8bf3b6254b30a7c218d32ac49451 2024-12-11T20:34:57,643 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/C of 786cab87062764a85ac24e1e3653be4a into 596c8bf3b6254b30a7c218d32ac49451(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:57,643 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:57,643 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/C, priority=11, startTime=1733949296681; duration=0sec 2024-12-11T20:34:57,643 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:57,643 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:C 2024-12-11T20:34:57,643 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:57,648 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:57,648 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/B is initiating minor compaction (all files) 2024-12-11T20:34:57,648 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/B in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:57,648 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/dd33786d07924e6fa2b7d5974ce41b8e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/9d0986c4828a4e338d47c627b656418f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2444f302d3c445348da5adad705d4f05] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=36.6 K 2024-12-11T20:34:57,649 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting dd33786d07924e6fa2b7d5974ce41b8e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733949295077 2024-12-11T20:34:57,651 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d0986c4828a4e338d47c627b656418f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733949295734 2024-12-11T20:34:57,651 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 2444f302d3c445348da5adad705d4f05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733949296914 2024-12-11T20:34:57,660 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:57,665 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#B#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:57,666 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/3bf4a144218e47609cdf3fdf5a0fea10 is 50, key is test_row_0/B:col10/1733949296914/Put/seqid=0 2024-12-11T20:34:57,667 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121113035e80e528430b8d37e5c86c38702a_786cab87062764a85ac24e1e3653be4a store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:57,669 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121113035e80e528430b8d37e5c86c38702a_786cab87062764a85ac24e1e3653be4a, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:57,669 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121113035e80e528430b8d37e5c86c38702a_786cab87062764a85ac24e1e3653be4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:57,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742039_1215 (size=13017) 2024-12-11T20:34:57,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742040_1216 (size=4469) 2024-12-11T20:34:57,702 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#A#compaction#177 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:57,703 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/57fff3fa25944254803ef852854037de is 175, key is test_row_0/A:col10/1733949296914/Put/seqid=0 2024-12-11T20:34:57,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742041_1217 (size=31971) 2024-12-11T20:34:57,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-11T20:34:57,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:57,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:57,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:57,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:57,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:57,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-11T20:34:57,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949357767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949357776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:57,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949357776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:57,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:57,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/9dfdc5bd18ef4c168c9128146d6719db 2024-12-11T20:34:57,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/8f24368e71f749b0bd6f87055c75d7fb is 50, key is test_row_0/B:col10/1733949297390/Put/seqid=0 2024-12-11T20:34:57,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742042_1218 (size=12301) 2024-12-11T20:34:57,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/8f24368e71f749b0bd6f87055c75d7fb 2024-12-11T20:34:57,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/7d868fe3e9444289ae03a555709ab5c8 is 50, key is test_row_0/C:col10/1733949297390/Put/seqid=0 2024-12-11T20:34:57,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-11T20:34:57,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:57,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:57,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:57,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:57,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:57,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:34:57,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742043_1219 (size=12301) 2024-12-11T20:34:57,880 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/7d868fe3e9444289ae03a555709ab5c8 2024-12-11T20:34:57,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/9dfdc5bd18ef4c168c9128146d6719db as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/9dfdc5bd18ef4c168c9128146d6719db 2024-12-11T20:34:57,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/9dfdc5bd18ef4c168c9128146d6719db, entries=150, sequenceid=350, filesize=30.5 K 2024-12-11T20:34:57,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/8f24368e71f749b0bd6f87055c75d7fb as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/8f24368e71f749b0bd6f87055c75d7fb 2024-12-11T20:34:57,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/8f24368e71f749b0bd6f87055c75d7fb, entries=150, sequenceid=350, filesize=12.0 K 2024-12-11T20:34:57,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/7d868fe3e9444289ae03a555709ab5c8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7d868fe3e9444289ae03a555709ab5c8 2024-12-11T20:34:57,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7d868fe3e9444289ae03a555709ab5c8, entries=150, sequenceid=350, filesize=12.0 K 2024-12-11T20:34:57,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 786cab87062764a85ac24e1e3653be4a in 517ms, sequenceid=350, compaction requested=true 2024-12-11T20:34:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:A, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:34:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T20:34:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:34:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-11T20:34:58,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-11T20:34:58,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:58,028 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T20:34:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:58,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:58,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d086aae291c442ac89115399c86b3a2f_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949297450/Put/seqid=0 2024-12-11T20:34:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-11T20:34:58,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742044_1220 (size=12454) 2024-12-11T20:34:58,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:58,054 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d086aae291c442ac89115399c86b3a2f_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d086aae291c442ac89115399c86b3a2f_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:58,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/67c9355224984af69a593e20ec00f2f7, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:58,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/67c9355224984af69a593e20ec00f2f7 is 175, key is test_row_0/A:col10/1733949297450/Put/seqid=0 2024-12-11T20:34:58,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742045_1221 (size=31255) 2024-12-11T20:34:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:58,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:34:58,098 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/3bf4a144218e47609cdf3fdf5a0fea10 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/3bf4a144218e47609cdf3fdf5a0fea10 2024-12-11T20:34:58,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949358097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949358099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,105 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/B of 786cab87062764a85ac24e1e3653be4a into 3bf4a144218e47609cdf3fdf5a0fea10(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:58,105 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:58,105 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/B, priority=13, startTime=1733949297284; duration=0sec 2024-12-11T20:34:58,105 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-11T20:34:58,105 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:B 2024-12-11T20:34:58,105 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-12-11T20:34:58,106 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-11T20:34:58,106 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-11T20:34:58,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,106 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. because compaction request was cancelled 2024-12-11T20:34:58,106 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:A 2024-12-11T20:34:58,106 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:B 2024-12-11T20:34:58,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949358100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,106 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:34:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,107 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:34:58,107 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/C is initiating minor compaction (all files) 2024-12-11T20:34:58,107 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/C in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:58,107 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/596c8bf3b6254b30a7c218d32ac49451, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/861ae5b4deda454592bd3dd2e49aa510, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7d868fe3e9444289ae03a555709ab5c8] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=36.7 K 2024-12-11T20:34:58,108 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 596c8bf3b6254b30a7c218d32ac49451, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733949295734 2024-12-11T20:34:58,108 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 861ae5b4deda454592bd3dd2e49aa510, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733949296914 2024-12-11T20:34:58,109 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d868fe3e9444289ae03a555709ab5c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733949297388 2024-12-11T20:34:58,119 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#C#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:34:58,120 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/2e15a60f893c42c9a5d9c129b25b5cab is 50, key is test_row_0/C:col10/1733949297390/Put/seqid=0 2024-12-11T20:34:58,123 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/57fff3fa25944254803ef852854037de as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/57fff3fa25944254803ef852854037de 2024-12-11T20:34:58,129 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/A of 786cab87062764a85ac24e1e3653be4a into 57fff3fa25944254803ef852854037de(size=31.2 K), total size for store is 61.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:58,129 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:58,129 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/A, priority=13, startTime=1733949297284; duration=0sec 2024-12-11T20:34:58,129 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:34:58,129 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:A 2024-12-11T20:34:58,129 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-11T20:34:58,130 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-11T20:34:58,130 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-11T20:34:58,130 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. because compaction request was cancelled 2024-12-11T20:34:58,130 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:B 2024-12-11T20:34:58,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742046_1222 (size=13051) 2024-12-11T20:34:58,153 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/2e15a60f893c42c9a5d9c129b25b5cab as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2e15a60f893c42c9a5d9c129b25b5cab 2024-12-11T20:34:58,161 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/C of 786cab87062764a85ac24e1e3653be4a into 2e15a60f893c42c9a5d9c129b25b5cab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:34:58,161 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:58,161 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/C, priority=13, startTime=1733949297913; duration=0sec 2024-12-11T20:34:58,161 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:34:58,161 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:C 2024-12-11T20:34:58,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949358207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949358207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949358217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,220 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949358417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949358417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949358427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,461 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/67c9355224984af69a593e20ec00f2f7 2024-12-11T20:34:58,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/276b4209494049ddb8ac92216a2c77d3 is 50, key is test_row_0/B:col10/1733949297450/Put/seqid=0 2024-12-11T20:34:58,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742047_1223 (size=12301) 2024-12-11T20:34:58,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-11T20:34:58,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949358727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949358727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949358737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:58,889 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/276b4209494049ddb8ac92216a2c77d3 2024-12-11T20:34:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/276114401d9143578c97bdb8b66a7d19 is 50, key is test_row_0/C:col10/1733949297450/Put/seqid=0 2024-12-11T20:34:58,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742048_1224 (size=12301) 2024-12-11T20:34:59,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949359236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:59,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949359247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:59,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:34:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:59,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949359247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:34:59,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:59,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:59,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:34:59,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:34:59,333 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/276114401d9143578c97bdb8b66a7d19 2024-12-11T20:34:59,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/67c9355224984af69a593e20ec00f2f7 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/67c9355224984af69a593e20ec00f2f7 2024-12-11T20:34:59,344 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/67c9355224984af69a593e20ec00f2f7, entries=150, sequenceid=370, filesize=30.5 K 2024-12-11T20:34:59,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/276b4209494049ddb8ac92216a2c77d3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/276b4209494049ddb8ac92216a2c77d3 2024-12-11T20:34:59,350 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/276b4209494049ddb8ac92216a2c77d3, entries=150, sequenceid=370, filesize=12.0 K 2024-12-11T20:34:59,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/276114401d9143578c97bdb8b66a7d19 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/276114401d9143578c97bdb8b66a7d19 2024-12-11T20:34:59,357 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/276114401d9143578c97bdb8b66a7d19, entries=150, sequenceid=370, filesize=12.0 K 2024-12-11T20:34:59,358 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 786cab87062764a85ac24e1e3653be4a in 1330ms, sequenceid=370, compaction requested=true 2024-12-11T20:34:59,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:34:59,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:59,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-11T20:34:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-11T20:34:59,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-11T20:34:59,366 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9520 sec 2024-12-11T20:34:59,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.9590 sec 2024-12-11T20:34:59,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-11T20:34:59,557 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:34:59,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:34:59,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-11T20:34:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-11T20:34:59,560 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:34:59,561 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:34:59,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:34:59,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-11T20:34:59,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-11T20:34:59,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:34:59,714 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-11T20:34:59,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:34:59,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:59,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:34:59,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:59,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:34:59,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:34:59,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115e1bcf1a83694a80937babe934fbc2ff_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949298098/Put/seqid=0 2024-12-11T20:34:59,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742049_1225 (size=12454) 2024-12-11T20:34:59,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:34:59,783 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115e1bcf1a83694a80937babe934fbc2ff_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115e1bcf1a83694a80937babe934fbc2ff_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:34:59,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/f41a53e74024458d98af4d3c0582fcf3, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:34:59,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/f41a53e74024458d98af4d3c0582fcf3 is 175, key is test_row_0/A:col10/1733949298098/Put/seqid=0 2024-12-11T20:34:59,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742050_1226 (size=31255) 2024-12-11T20:34:59,821 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/f41a53e74024458d98af4d3c0582fcf3 2024-12-11T20:34:59,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/194ae883b4054b4b89716bdbcea37ecb is 50, key is test_row_0/B:col10/1733949298098/Put/seqid=0 2024-12-11T20:34:59,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742051_1227 (size=12301) 2024-12-11T20:34:59,878 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/194ae883b4054b4b89716bdbcea37ecb 2024-12-11T20:34:59,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-11T20:34:59,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/284c16451f814f31b52a2311d0bbaf0d is 50, key is test_row_0/C:col10/1733949298098/Put/seqid=0 2024-12-11T20:34:59,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742052_1228 (size=12301) 2024-12-11T20:35:00,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-11T20:35:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:00,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:35:00,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949360287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949360288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949360290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,293 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,337 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/284c16451f814f31b52a2311d0bbaf0d 2024-12-11T20:35:00,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/f41a53e74024458d98af4d3c0582fcf3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f41a53e74024458d98af4d3c0582fcf3 2024-12-11T20:35:00,356 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f41a53e74024458d98af4d3c0582fcf3, entries=150, sequenceid=387, filesize=30.5 K 2024-12-11T20:35:00,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/194ae883b4054b4b89716bdbcea37ecb as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/194ae883b4054b4b89716bdbcea37ecb 2024-12-11T20:35:00,362 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/194ae883b4054b4b89716bdbcea37ecb, entries=150, sequenceid=387, filesize=12.0 K 2024-12-11T20:35:00,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/284c16451f814f31b52a2311d0bbaf0d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/284c16451f814f31b52a2311d0bbaf0d 2024-12-11T20:35:00,376 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/284c16451f814f31b52a2311d0bbaf0d, entries=150, sequenceid=387, filesize=12.0 K 2024-12-11T20:35:00,378 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 786cab87062764a85ac24e1e3653be4a in 664ms, sequenceid=387, compaction requested=true 2024-12-11T20:35:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:35:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:00,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-11T20:35:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-11T20:35:00,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-11T20:35:00,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 819 msec 2024-12-11T20:35:00,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 828 msec 2024-12-11T20:35:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:00,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-11T20:35:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:35:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:35:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:35:00,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:00,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121120bed42782044e4b986b6310057a8be7_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949300283/Put/seqid=0 2024-12-11T20:35:00,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949360415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949360416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949360421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742053_1229 (size=14994) 2024-12-11T20:35:00,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949360527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949360527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949360536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-11T20:35:00,697 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:00,697 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-11T20:35:00,699 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T20:35:00,701 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:00,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949360737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949360737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949360746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T20:35:00,840 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:00,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T20:35:00,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:00,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:35:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:00,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:00,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:00,861 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121120bed42782044e4b986b6310057a8be7_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121120bed42782044e4b986b6310057a8be7_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:00,862 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/313189f7d5014599bad1656395ec536b, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:35:00,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/313189f7d5014599bad1656395ec536b is 175, key is test_row_0/A:col10/1733949300283/Put/seqid=0 2024-12-11T20:35:00,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742054_1230 (size=39949) 2024-12-11T20:35:01,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T20:35:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:35:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T20:35:01,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949361046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:01,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:01,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949361046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:01,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949361057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:01,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:01,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T20:35:01,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:35:01,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:01,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49746 deadline: 1733949361176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:01,177 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_0' in 786cab87062764a85ac24e1e3653be4a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:01,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:01,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49720 deadline: 1733949361187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:01,190 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 786cab87062764a85ac24e1e3653be4a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18293 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:01,267 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/313189f7d5014599bad1656395ec536b 2024-12-11T20:35:01,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/83b9fda79d604f998e06b896efcbda46 is 50, key is test_row_0/B:col10/1733949300283/Put/seqid=0 2024-12-11T20:35:01,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742055_1231 (size=12301) 2024-12-11T20:35:01,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/83b9fda79d604f998e06b896efcbda46 2024-12-11T20:35:01,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/ca89036a1aa34e339ec7102b5ab56e7d is 50, key is test_row_0/C:col10/1733949300283/Put/seqid=0 2024-12-11T20:35:01,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742056_1232 (size=12301) 2024-12-11T20:35:01,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T20:35:01,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:35:01,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T20:35:01,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T20:35:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:35:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49776 deadline: 1733949361556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:01,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49740 deadline: 1733949361556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:01,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49764 deadline: 1733949361567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:01,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:01,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T20:35:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. as already flushing 2024-12-11T20:35:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:01,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/ca89036a1aa34e339ec7102b5ab56e7d 2024-12-11T20:35:01,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/313189f7d5014599bad1656395ec536b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/313189f7d5014599bad1656395ec536b 2024-12-11T20:35:01,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/313189f7d5014599bad1656395ec536b, entries=200, sequenceid=406, filesize=39.0 K 2024-12-11T20:35:01,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/83b9fda79d604f998e06b896efcbda46 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/83b9fda79d604f998e06b896efcbda46 2024-12-11T20:35:01,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/83b9fda79d604f998e06b896efcbda46, entries=150, sequenceid=406, filesize=12.0 K 2024-12-11T20:35:01,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/ca89036a1aa34e339ec7102b5ab56e7d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ca89036a1aa34e339ec7102b5ab56e7d 2024-12-11T20:35:01,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ca89036a1aa34e339ec7102b5ab56e7d, entries=150, sequenceid=406, filesize=12.0 K 2024-12-11T20:35:01,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 786cab87062764a85ac24e1e3653be4a in 1344ms, sequenceid=406, compaction requested=true 2024-12-11T20:35:01,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:35:01,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:01,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:01,745 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-11T20:35:01,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:01,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:01,745 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-11T20:35:01,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 786cab87062764a85ac24e1e3653be4a:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:01,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:01,749 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-11T20:35:01,749 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 165685 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-11T20:35:01,749 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/B is initiating minor compaction (all files) 2024-12-11T20:35:01,749 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/A is initiating minor compaction (all files) 2024-12-11T20:35:01,749 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/A in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,749 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/B in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,749 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/3bf4a144218e47609cdf3fdf5a0fea10, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/8f24368e71f749b0bd6f87055c75d7fb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/276b4209494049ddb8ac92216a2c77d3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/194ae883b4054b4b89716bdbcea37ecb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/83b9fda79d604f998e06b896efcbda46] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=60.8 K 2024-12-11T20:35:01,749 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/57fff3fa25944254803ef852854037de, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/9dfdc5bd18ef4c168c9128146d6719db, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/67c9355224984af69a593e20ec00f2f7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f41a53e74024458d98af4d3c0582fcf3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/313189f7d5014599bad1656395ec536b] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=161.8 K 2024-12-11T20:35:01,749 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,750 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/57fff3fa25944254803ef852854037de, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/9dfdc5bd18ef4c168c9128146d6719db, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/67c9355224984af69a593e20ec00f2f7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f41a53e74024458d98af4d3c0582fcf3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/313189f7d5014599bad1656395ec536b] 2024-12-11T20:35:01,750 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bf4a144218e47609cdf3fdf5a0fea10, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733949296914 2024-12-11T20:35:01,750 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57fff3fa25944254803ef852854037de, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1733949296914 2024-12-11T20:35:01,750 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f24368e71f749b0bd6f87055c75d7fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733949297388 2024-12-11T20:35:01,751 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dfdc5bd18ef4c168c9128146d6719db, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733949297388 2024-12-11T20:35:01,751 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 276b4209494049ddb8ac92216a2c77d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733949297433 2024-12-11T20:35:01,751 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 194ae883b4054b4b89716bdbcea37ecb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733949298094 2024-12-11T20:35:01,751 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67c9355224984af69a593e20ec00f2f7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733949297433 2024-12-11T20:35:01,751 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 83b9fda79d604f998e06b896efcbda46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733949300283 2024-12-11T20:35:01,751 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting f41a53e74024458d98af4d3c0582fcf3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733949298094 2024-12-11T20:35:01,752 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 313189f7d5014599bad1656395ec536b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733949300283 2024-12-11T20:35:01,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-11T20:35:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:01,777 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T20:35:01,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:35:01,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:01,778 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:35:01,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:35:01,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:01,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:35:01,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:01,783 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#B#compaction#192 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:01,783 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211fbdc57966f044ccca7b82af5981936ae_786cab87062764a85ac24e1e3653be4a store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:35:01,783 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/aa3e7bee47a24077a8b9681bbc742d21 is 50, key is test_row_0/B:col10/1733949300283/Put/seqid=0 2024-12-11T20:35:01,785 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211fbdc57966f044ccca7b82af5981936ae_786cab87062764a85ac24e1e3653be4a, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:35:01,785 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211fbdc57966f044ccca7b82af5981936ae_786cab87062764a85ac24e1e3653be4a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:35:01,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412114315442579f149eeb8176bbe93f7b7bf_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949300414/Put/seqid=0 2024-12-11T20:35:01,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742057_1233 (size=4469) 2024-12-11T20:35:01,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742059_1235 (size=12454) 2024-12-11T20:35:01,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742058_1234 (size=13187) 2024-12-11T20:35:01,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:01,826 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412114315442579f149eeb8176bbe93f7b7bf_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114315442579f149eeb8176bbe93f7b7bf_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:01,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/e72478da9d1e442496fcf9207bc01b27, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:35:01,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/e72478da9d1e442496fcf9207bc01b27 is 175, key is test_row_0/A:col10/1733949300414/Put/seqid=0 2024-12-11T20:35:01,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T20:35:01,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742060_1236 (size=31255) 2024-12-11T20:35:01,841 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=419, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/e72478da9d1e442496fcf9207bc01b27 2024-12-11T20:35:01,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/ea9760055dde4479a88840669457cd6c is 50, key is test_row_0/B:col10/1733949300414/Put/seqid=0 2024-12-11T20:35:01,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742061_1237 (size=12301) 2024-12-11T20:35:01,872 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/ea9760055dde4479a88840669457cd6c 2024-12-11T20:35:01,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/764c38aab6e841b393d8f18c79e9bbf2 is 50, key is test_row_0/C:col10/1733949300414/Put/seqid=0 2024-12-11T20:35:01,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742062_1238 (size=12301) 2024-12-11T20:35:02,217 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#A#compaction#191 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:02,217 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/73e818b44187452c9d0cec06f5a9b937 is 175, key is test_row_0/A:col10/1733949300283/Put/seqid=0 2024-12-11T20:35:02,229 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/aa3e7bee47a24077a8b9681bbc742d21 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/aa3e7bee47a24077a8b9681bbc742d21 2024-12-11T20:35:02,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742063_1239 (size=32141) 2024-12-11T20:35:02,239 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/B of 786cab87062764a85ac24e1e3653be4a into aa3e7bee47a24077a8b9681bbc742d21(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:02,239 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:35:02,240 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/B, priority=11, startTime=1733949301745; duration=0sec 2024-12-11T20:35:02,240 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:02,240 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:B 2024-12-11T20:35:02,240 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:35:02,242 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:35:02,242 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 786cab87062764a85ac24e1e3653be4a/C is initiating minor compaction (all files) 2024-12-11T20:35:02,242 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 786cab87062764a85ac24e1e3653be4a/C in TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:02,242 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2e15a60f893c42c9a5d9c129b25b5cab, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/276114401d9143578c97bdb8b66a7d19, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/284c16451f814f31b52a2311d0bbaf0d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ca89036a1aa34e339ec7102b5ab56e7d] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp, totalSize=48.8 K 2024-12-11T20:35:02,244 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e15a60f893c42c9a5d9c129b25b5cab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733949297388 2024-12-11T20:35:02,244 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 276114401d9143578c97bdb8b66a7d19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1733949297433 2024-12-11T20:35:02,245 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 284c16451f814f31b52a2311d0bbaf0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1733949298094 2024-12-11T20:35:02,245 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting ca89036a1aa34e339ec7102b5ab56e7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1733949300283 2024-12-11T20:35:02,257 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 786cab87062764a85ac24e1e3653be4a#C#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:02,258 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/57e138b97d084e36a5719ff15d3df355 is 50, key is test_row_0/C:col10/1733949300283/Put/seqid=0 2024-12-11T20:35:02,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742064_1240 (size=13187) 2024-12-11T20:35:02,294 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/764c38aab6e841b393d8f18c79e9bbf2 2024-12-11T20:35:02,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/e72478da9d1e442496fcf9207bc01b27 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/e72478da9d1e442496fcf9207bc01b27 2024-12-11T20:35:02,311 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/e72478da9d1e442496fcf9207bc01b27, entries=150, sequenceid=419, filesize=30.5 K 2024-12-11T20:35:02,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/ea9760055dde4479a88840669457cd6c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/ea9760055dde4479a88840669457cd6c 2024-12-11T20:35:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,319 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/ea9760055dde4479a88840669457cd6c, entries=150, sequenceid=419, filesize=12.0 K 2024-12-11T20:35:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/764c38aab6e841b393d8f18c79e9bbf2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/764c38aab6e841b393d8f18c79e9bbf2 2024-12-11T20:35:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,326 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/764c38aab6e841b393d8f18c79e9bbf2, entries=150, sequenceid=419, filesize=12.0 K 2024-12-11T20:35:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,327 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 786cab87062764a85ac24e1e3653be4a in 549ms, sequenceid=419, compaction requested=false 2024-12-11T20:35:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:35:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:02,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-11T20:35:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-11T20:35:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-11T20:35:02,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6280 sec 2024-12-11T20:35:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.6350 sec 2024-12-11T20:35:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,418 INFO [Thread-657 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-657. 2024-12-11T20:35:02,418 DEBUG [Thread-657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:02,418 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,418 DEBUG [Thread-657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:02,419 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-12-11T20:35:02,419 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:02,419 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,419 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,419 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,421 INFO [Thread-655 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-655. 2024-12-11T20:35:02,421 DEBUG [Thread-655 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:02,421 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,421 DEBUG [Thread-655 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,421 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:02,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:02,449 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-12-11T20:35:02,450 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:02,450 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,450 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,450 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:02,569 INFO [Thread-644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-644. 2024-12-11T20:35:02,569 DEBUG [Thread-644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:02,569 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,569 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:02,570 INFO [Thread-642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-642. 2024-12-11T20:35:02,570 DEBUG [Thread-642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:02,570 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,570 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:02,578 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-12-11T20:35:02,578 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:02,578 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,578 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:02,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:02,644 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/73e818b44187452c9d0cec06f5a9b937 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/73e818b44187452c9d0cec06f5a9b937 2024-12-11T20:35:02,650 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/A of 786cab87062764a85ac24e1e3653be4a into 73e818b44187452c9d0cec06f5a9b937(size=31.4 K), total size for store is 61.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:02,650 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:35:02,650 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/A, priority=11, startTime=1733949301745; duration=0sec 2024-12-11T20:35:02,650 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:02,650 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:A 2024-12-11T20:35:02,699 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/57e138b97d084e36a5719ff15d3df355 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/57e138b97d084e36a5719ff15d3df355 2024-12-11T20:35:02,706 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 786cab87062764a85ac24e1e3653be4a/C of 786cab87062764a85ac24e1e3653be4a into 57e138b97d084e36a5719ff15d3df355(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:02,707 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 786cab87062764a85ac24e1e3653be4a: 2024-12-11T20:35:02,707 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., storeName=786cab87062764a85ac24e1e3653be4a/C, priority=12, startTime=1733949301745; duration=0sec 2024-12-11T20:35:02,707 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:02,707 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 786cab87062764a85ac24e1e3653be4a:C 2024-12-11T20:35:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-11T20:35:02,848 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:07,172 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T20:35:11,226 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-12-11T20:35:11,226 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:11,226 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:11,226 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:11,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:11,248 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-12-11T20:35:11,248 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:11,248 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:11,248 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:11,248 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:11,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T20:35:11,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-12-11T20:35:11,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-12-11T20:35:11,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-12-11T20:35:11,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-12-11T20:35:11,248 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-12-11T20:35:11,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T20:35:11,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3100 2024-12-11T20:35:11,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3312 2024-12-11T20:35:11,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T20:35:11,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1382 2024-12-11T20:35:11,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4146 rows 2024-12-11T20:35:11,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1389 2024-12-11T20:35:11,249 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4164 rows 2024-12-11T20:35:11,249 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T20:35:11,249 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:35:11,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:11,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:11,249 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:11,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-11T20:35:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:11,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T20:35:11,253 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949311253"}]},"ts":"1733949311253"} 2024-12-11T20:35:11,255 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T20:35:11,255 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T20:35:11,256 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T20:35:11,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, UNASSIGN}] 2024-12-11T20:35:11,258 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, UNASSIGN 2024-12-11T20:35:11,258 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=786cab87062764a85ac24e1e3653be4a, regionState=CLOSING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:11,260 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, UNASSIGN because future has completed 2024-12-11T20:35:11,260 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T20:35:11,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:35:11,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T20:35:11,413 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:11,413 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T20:35:11,413 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 786cab87062764a85ac24e1e3653be4a, disabling compactions & flushes 2024-12-11T20:35:11,413 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:11,413 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:11,413 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. after waiting 0 ms 2024-12-11T20:35:11,413 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:11,413 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 786cab87062764a85ac24e1e3653be4a 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-11T20:35:11,414 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=A 2024-12-11T20:35:11,414 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:11,414 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=B 2024-12-11T20:35:11,414 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:11,414 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 786cab87062764a85ac24e1e3653be4a, store=C 2024-12-11T20:35:11,414 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:11,425 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412110c1bb5936916495b985695ccf61e4083_786cab87062764a85ac24e1e3653be4a is 50, key is test_row_0/A:col10/1733949311217/Put/seqid=0 2024-12-11T20:35:11,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742065_1241 (size=12454) 2024-12-11T20:35:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T20:35:11,833 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:11,838 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412110c1bb5936916495b985695ccf61e4083_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412110c1bb5936916495b985695ccf61e4083_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:11,839 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/85796bd3716347e68f7cab91ffa2e3c9, store: [table=TestAcidGuarantees family=A region=786cab87062764a85ac24e1e3653be4a] 2024-12-11T20:35:11,839 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/85796bd3716347e68f7cab91ffa2e3c9 is 175, key is test_row_0/A:col10/1733949311217/Put/seqid=0 2024-12-11T20:35:11,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742066_1242 (size=31255) 2024-12-11T20:35:11,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T20:35:12,243 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=430, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/85796bd3716347e68f7cab91ffa2e3c9 2024-12-11T20:35:12,250 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/80561e68b4f243a69326efa4f4a6e8d5 is 50, key is test_row_0/B:col10/1733949311217/Put/seqid=0 2024-12-11T20:35:12,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742067_1243 (size=12301) 2024-12-11T20:35:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T20:35:12,656 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/80561e68b4f243a69326efa4f4a6e8d5 2024-12-11T20:35:12,666 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/32bb15c846034997b21f5e33e982d9b6 is 50, key is test_row_0/C:col10/1733949311217/Put/seqid=0 2024-12-11T20:35:12,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742068_1244 (size=12301) 2024-12-11T20:35:13,071 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/32bb15c846034997b21f5e33e982d9b6 2024-12-11T20:35:13,076 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/A/85796bd3716347e68f7cab91ffa2e3c9 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/85796bd3716347e68f7cab91ffa2e3c9 2024-12-11T20:35:13,079 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/85796bd3716347e68f7cab91ffa2e3c9, entries=150, sequenceid=430, filesize=30.5 K 2024-12-11T20:35:13,080 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/B/80561e68b4f243a69326efa4f4a6e8d5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/80561e68b4f243a69326efa4f4a6e8d5 2024-12-11T20:35:13,084 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/80561e68b4f243a69326efa4f4a6e8d5, entries=150, sequenceid=430, filesize=12.0 K 2024-12-11T20:35:13,085 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/.tmp/C/32bb15c846034997b21f5e33e982d9b6 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/32bb15c846034997b21f5e33e982d9b6 2024-12-11T20:35:13,089 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/32bb15c846034997b21f5e33e982d9b6, entries=150, sequenceid=430, filesize=12.0 K 2024-12-11T20:35:13,090 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 786cab87062764a85ac24e1e3653be4a in 1676ms, sequenceid=430, compaction requested=true 2024-12-11T20:35:13,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7fa9982e4ab5451580ebfa5736044023, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/988bb673d2134f5491ed3832ce1ed893, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a0745668bce4e0c8295c84200613e7f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/918e97d1756c4fb58643d6a9686b273d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ff81c210480142e8a14f985d043fbab5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a1edcf5aabe48c18fdcf96679023022, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/18add2e22f054885a34e7ff1f93a8fe3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/548d283de6ee4334a96ccb1e22c0e259, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/e65118c32d9540b796c591ed7aaf062f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7c08166792ca4cb28d772c041a762c4d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/273466f393de49808e4e0f0d68e8c638, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/024c81c211fa4a6facb9b3d172a0959d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ca35f0db55564127b719ef64c5fb3f1c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/8960bd7a47ec4ec8915814380255e62a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/63d29f522ef5480faac63eb6aaeb7eba, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f6adf086074148bcafe168a22bc89347, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/863ada196d5249bfbce3e50726ca0245, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7f2fe3e54f634e6b92ebc48c5088a8af, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/0bebb45622c54783a20d0b5eae605eb1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/c225980247d84d2e98fc567115550c13, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/57fff3fa25944254803ef852854037de, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/a3de92896d4d458b8511f3590384bd49, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/9dfdc5bd18ef4c168c9128146d6719db, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/67c9355224984af69a593e20ec00f2f7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f41a53e74024458d98af4d3c0582fcf3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/313189f7d5014599bad1656395ec536b] to archive 2024-12-11T20:35:13,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:35:13,094 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/988bb673d2134f5491ed3832ce1ed893 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/988bb673d2134f5491ed3832ce1ed893 2024-12-11T20:35:13,094 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7fa9982e4ab5451580ebfa5736044023 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7fa9982e4ab5451580ebfa5736044023 2024-12-11T20:35:13,094 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a0745668bce4e0c8295c84200613e7f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a0745668bce4e0c8295c84200613e7f 2024-12-11T20:35:13,094 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/918e97d1756c4fb58643d6a9686b273d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/918e97d1756c4fb58643d6a9686b273d 2024-12-11T20:35:13,095 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ff81c210480142e8a14f985d043fbab5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ff81c210480142e8a14f985d043fbab5 2024-12-11T20:35:13,096 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a1edcf5aabe48c18fdcf96679023022 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/2a1edcf5aabe48c18fdcf96679023022 2024-12-11T20:35:13,097 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/18add2e22f054885a34e7ff1f93a8fe3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/18add2e22f054885a34e7ff1f93a8fe3 2024-12-11T20:35:13,097 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7c08166792ca4cb28d772c041a762c4d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7c08166792ca4cb28d772c041a762c4d 2024-12-11T20:35:13,097 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/e65118c32d9540b796c591ed7aaf062f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/e65118c32d9540b796c591ed7aaf062f 2024-12-11T20:35:13,097 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/273466f393de49808e4e0f0d68e8c638 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/273466f393de49808e4e0f0d68e8c638 2024-12-11T20:35:13,097 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/548d283de6ee4334a96ccb1e22c0e259 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/548d283de6ee4334a96ccb1e22c0e259 2024-12-11T20:35:13,098 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/024c81c211fa4a6facb9b3d172a0959d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/024c81c211fa4a6facb9b3d172a0959d 2024-12-11T20:35:13,098 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ca35f0db55564127b719ef64c5fb3f1c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/ca35f0db55564127b719ef64c5fb3f1c 2024-12-11T20:35:13,099 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/63d29f522ef5480faac63eb6aaeb7eba to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/63d29f522ef5480faac63eb6aaeb7eba 2024-12-11T20:35:13,099 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7f2fe3e54f634e6b92ebc48c5088a8af to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/7f2fe3e54f634e6b92ebc48c5088a8af 2024-12-11T20:35:13,100 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/0bebb45622c54783a20d0b5eae605eb1 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/0bebb45622c54783a20d0b5eae605eb1 2024-12-11T20:35:13,100 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f6adf086074148bcafe168a22bc89347 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f6adf086074148bcafe168a22bc89347 2024-12-11T20:35:13,101 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/8960bd7a47ec4ec8915814380255e62a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/8960bd7a47ec4ec8915814380255e62a 2024-12-11T20:35:13,102 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/c225980247d84d2e98fc567115550c13 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/c225980247d84d2e98fc567115550c13 2024-12-11T20:35:13,102 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/9dfdc5bd18ef4c168c9128146d6719db to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/9dfdc5bd18ef4c168c9128146d6719db 2024-12-11T20:35:13,102 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/863ada196d5249bfbce3e50726ca0245 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/863ada196d5249bfbce3e50726ca0245 2024-12-11T20:35:13,102 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/57fff3fa25944254803ef852854037de to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/57fff3fa25944254803ef852854037de 2024-12-11T20:35:13,102 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/a3de92896d4d458b8511f3590384bd49 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/a3de92896d4d458b8511f3590384bd49 2024-12-11T20:35:13,103 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/67c9355224984af69a593e20ec00f2f7 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/67c9355224984af69a593e20ec00f2f7 2024-12-11T20:35:13,103 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/313189f7d5014599bad1656395ec536b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/313189f7d5014599bad1656395ec536b 2024-12-11T20:35:13,103 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f41a53e74024458d98af4d3c0582fcf3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/f41a53e74024458d98af4d3c0582fcf3 2024-12-11T20:35:13,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/0352db70221c49bc9f6e09a3830569cd, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/1f3d352946ab4731a2b9adc933b574f8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/ac48707e4c1048ccb67a20bbf8123f65, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/0c1f51ab3dc14e1c82f0f44463c1f403, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/02cfaddc47404fa7abaf39325843968b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/d715ca0f00a0427cad320d9eddbf685b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/498dc3d191ea4cbba36ada571a10c664, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/57bc7405d2fd4f93aa46094851292876, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73eb0920bfd043e589bb6d6e6e1f3dbe, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/8a80a90be8bf49c18df7a4a25fa88b26, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2a1752087e06430f8393ef9e7eda2cbf, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/31ec2ec8f3ee40bd94606cad344f3786, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/abdfb84856304d3b82828361d2567af6, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/e6b68ef3b00f40048d71eeaccad96ad8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bab7b44613d648208c4db9d8d839e8b4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/5fae95f87dcc4c9b92a0596581cdb99d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bb7fdf6ed3c548e999ddac71fcc198a7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/dd33786d07924e6fa2b7d5974ce41b8e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73ba9abb716840468769de497090effb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/9d0986c4828a4e338d47c627b656418f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/3bf4a144218e47609cdf3fdf5a0fea10, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2444f302d3c445348da5adad705d4f05, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/8f24368e71f749b0bd6f87055c75d7fb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/276b4209494049ddb8ac92216a2c77d3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/194ae883b4054b4b89716bdbcea37ecb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/83b9fda79d604f998e06b896efcbda46] to archive 2024-12-11T20:35:13,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:35:13,107 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/02cfaddc47404fa7abaf39325843968b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/02cfaddc47404fa7abaf39325843968b 2024-12-11T20:35:13,107 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/d715ca0f00a0427cad320d9eddbf685b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/d715ca0f00a0427cad320d9eddbf685b 2024-12-11T20:35:13,108 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/57bc7405d2fd4f93aa46094851292876 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/57bc7405d2fd4f93aa46094851292876 2024-12-11T20:35:13,108 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/1f3d352946ab4731a2b9adc933b574f8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/1f3d352946ab4731a2b9adc933b574f8 2024-12-11T20:35:13,108 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/498dc3d191ea4cbba36ada571a10c664 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/498dc3d191ea4cbba36ada571a10c664 2024-12-11T20:35:13,108 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/0c1f51ab3dc14e1c82f0f44463c1f403 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/0c1f51ab3dc14e1c82f0f44463c1f403 2024-12-11T20:35:13,108 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/ac48707e4c1048ccb67a20bbf8123f65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/ac48707e4c1048ccb67a20bbf8123f65 2024-12-11T20:35:13,110 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/8a80a90be8bf49c18df7a4a25fa88b26 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/8a80a90be8bf49c18df7a4a25fa88b26 2024-12-11T20:35:13,110 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2a1752087e06430f8393ef9e7eda2cbf to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2a1752087e06430f8393ef9e7eda2cbf 2024-12-11T20:35:13,110 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73eb0920bfd043e589bb6d6e6e1f3dbe to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73eb0920bfd043e589bb6d6e6e1f3dbe 2024-12-11T20:35:13,111 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/31ec2ec8f3ee40bd94606cad344f3786 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/31ec2ec8f3ee40bd94606cad344f3786 2024-12-11T20:35:13,111 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/e6b68ef3b00f40048d71eeaccad96ad8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/e6b68ef3b00f40048d71eeaccad96ad8 2024-12-11T20:35:13,111 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/abdfb84856304d3b82828361d2567af6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/abdfb84856304d3b82828361d2567af6 2024-12-11T20:35:13,112 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bab7b44613d648208c4db9d8d839e8b4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bab7b44613d648208c4db9d8d839e8b4 2024-12-11T20:35:13,112 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/0352db70221c49bc9f6e09a3830569cd to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/0352db70221c49bc9f6e09a3830569cd 2024-12-11T20:35:13,112 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/5fae95f87dcc4c9b92a0596581cdb99d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/5fae95f87dcc4c9b92a0596581cdb99d 2024-12-11T20:35:13,112 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/dd33786d07924e6fa2b7d5974ce41b8e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/dd33786d07924e6fa2b7d5974ce41b8e 2024-12-11T20:35:13,112 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bb7fdf6ed3c548e999ddac71fcc198a7 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/bb7fdf6ed3c548e999ddac71fcc198a7 2024-12-11T20:35:13,113 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73ba9abb716840468769de497090effb to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/73ba9abb716840468769de497090effb 2024-12-11T20:35:13,113 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/9d0986c4828a4e338d47c627b656418f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/9d0986c4828a4e338d47c627b656418f 2024-12-11T20:35:13,114 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/276b4209494049ddb8ac92216a2c77d3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/276b4209494049ddb8ac92216a2c77d3 2024-12-11T20:35:13,114 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/8f24368e71f749b0bd6f87055c75d7fb to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/8f24368e71f749b0bd6f87055c75d7fb 2024-12-11T20:35:13,114 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/83b9fda79d604f998e06b896efcbda46 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/83b9fda79d604f998e06b896efcbda46 2024-12-11T20:35:13,114 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/3bf4a144218e47609cdf3fdf5a0fea10 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/3bf4a144218e47609cdf3fdf5a0fea10 2024-12-11T20:35:13,114 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/194ae883b4054b4b89716bdbcea37ecb to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/194ae883b4054b4b89716bdbcea37ecb 2024-12-11T20:35:13,114 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2444f302d3c445348da5adad705d4f05 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/2444f302d3c445348da5adad705d4f05 2024-12-11T20:35:13,119 DEBUG [StoreCloser-TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/fd0bbe52c5be4a2cba4b7e9dfcea426e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/e82fe2dd259c407b95bccb5cf6f7f288, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2a411e2c5f60487ea2a109d72e90f73b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/984a4796099a458eb9810e896fb8b040, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3ffc61f9517d4accbc3ad391005dc711, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7312334def524ed3a5cc191903d1c2b2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/d589015e4afe4fb689e4bc6c0b60a3b3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/aef40cb60d5a41028b5dfdee5e3ab763, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af14a1179e5a4aaaa632e48be85cbb00, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/40f8fec814fe4ff38de027ef27c81303, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/f256c897db8947cc9c6a19853c765efb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/c5d42630b7684f0ca4e5b837f74b9c30, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2392396b8c0847e1b0934e03b7f7df2a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3a3620d93ebc4d158ab89ad1206a3321, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ff084f09a2284ab58768ce58d45a8416, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/8fe7f046b505440d8073867975097c87, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af1f6a70c1094fc88922590c603a91f4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3fd406df92ba418eba7ebf0714a6f035, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/596c8bf3b6254b30a7c218d32ac49451, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/63902497595d4c8394814c830ca824d4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/861ae5b4deda454592bd3dd2e49aa510, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2e15a60f893c42c9a5d9c129b25b5cab, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7d868fe3e9444289ae03a555709ab5c8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/276114401d9143578c97bdb8b66a7d19, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/284c16451f814f31b52a2311d0bbaf0d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ca89036a1aa34e339ec7102b5ab56e7d] to archive 2024-12-11T20:35:13,120 DEBUG [StoreCloser-TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:35:13,122 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/aef40cb60d5a41028b5dfdee5e3ab763 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/aef40cb60d5a41028b5dfdee5e3ab763 2024-12-11T20:35:13,122 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3ffc61f9517d4accbc3ad391005dc711 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3ffc61f9517d4accbc3ad391005dc711 2024-12-11T20:35:13,122 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/fd0bbe52c5be4a2cba4b7e9dfcea426e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/fd0bbe52c5be4a2cba4b7e9dfcea426e 2024-12-11T20:35:13,122 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/e82fe2dd259c407b95bccb5cf6f7f288 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/e82fe2dd259c407b95bccb5cf6f7f288 2024-12-11T20:35:13,123 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/984a4796099a458eb9810e896fb8b040 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/984a4796099a458eb9810e896fb8b040 2024-12-11T20:35:13,123 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2a411e2c5f60487ea2a109d72e90f73b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2a411e2c5f60487ea2a109d72e90f73b 2024-12-11T20:35:13,123 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/d589015e4afe4fb689e4bc6c0b60a3b3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/d589015e4afe4fb689e4bc6c0b60a3b3 2024-12-11T20:35:13,123 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7312334def524ed3a5cc191903d1c2b2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7312334def524ed3a5cc191903d1c2b2 2024-12-11T20:35:13,124 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af14a1179e5a4aaaa632e48be85cbb00 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af14a1179e5a4aaaa632e48be85cbb00 2024-12-11T20:35:13,124 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3a3620d93ebc4d158ab89ad1206a3321 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3a3620d93ebc4d158ab89ad1206a3321 2024-12-11T20:35:13,124 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/40f8fec814fe4ff38de027ef27c81303 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/40f8fec814fe4ff38de027ef27c81303 2024-12-11T20:35:13,124 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2392396b8c0847e1b0934e03b7f7df2a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2392396b8c0847e1b0934e03b7f7df2a 2024-12-11T20:35:13,125 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/c5d42630b7684f0ca4e5b837f74b9c30 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/c5d42630b7684f0ca4e5b837f74b9c30 2024-12-11T20:35:13,125 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/f256c897db8947cc9c6a19853c765efb to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/f256c897db8947cc9c6a19853c765efb 2024-12-11T20:35:13,125 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/8fe7f046b505440d8073867975097c87 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/8fe7f046b505440d8073867975097c87 2024-12-11T20:35:13,125 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ff084f09a2284ab58768ce58d45a8416 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ff084f09a2284ab58768ce58d45a8416 2024-12-11T20:35:13,126 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/596c8bf3b6254b30a7c218d32ac49451 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/596c8bf3b6254b30a7c218d32ac49451 2024-12-11T20:35:13,127 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af1f6a70c1094fc88922590c603a91f4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/af1f6a70c1094fc88922590c603a91f4 2024-12-11T20:35:13,127 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/63902497595d4c8394814c830ca824d4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/63902497595d4c8394814c830ca824d4 2024-12-11T20:35:13,127 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3fd406df92ba418eba7ebf0714a6f035 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/3fd406df92ba418eba7ebf0714a6f035 2024-12-11T20:35:13,127 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2e15a60f893c42c9a5d9c129b25b5cab to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/2e15a60f893c42c9a5d9c129b25b5cab 2024-12-11T20:35:13,127 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/861ae5b4deda454592bd3dd2e49aa510 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/861ae5b4deda454592bd3dd2e49aa510 2024-12-11T20:35:13,127 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/276114401d9143578c97bdb8b66a7d19 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/276114401d9143578c97bdb8b66a7d19 2024-12-11T20:35:13,127 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7d868fe3e9444289ae03a555709ab5c8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/7d868fe3e9444289ae03a555709ab5c8 2024-12-11T20:35:13,128 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/284c16451f814f31b52a2311d0bbaf0d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/284c16451f814f31b52a2311d0bbaf0d 2024-12-11T20:35:13,128 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ca89036a1aa34e339ec7102b5ab56e7d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/ca89036a1aa34e339ec7102b5ab56e7d 2024-12-11T20:35:13,132 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/recovered.edits/433.seqid, newMaxSeqId=433, maxSeqId=4 2024-12-11T20:35:13,133 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. 2024-12-11T20:35:13,133 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 786cab87062764a85ac24e1e3653be4a: Waiting for close lock at 1733949311413Running coprocessor pre-close hooks at 1733949311413Disabling compacts and flushes for region at 1733949311413Disabling writes for close at 1733949311413Obtaining lock to block concurrent updates at 1733949311413Preparing flush snapshotting stores in 786cab87062764a85ac24e1e3653be4a at 1733949311413Finished memstore snapshotting TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733949311414 (+1 ms)Flushing stores of TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a. at 1733949311415 (+1 ms)Flushing 786cab87062764a85ac24e1e3653be4a/A: creating writer at 1733949311415Flushing mob file 786cab87062764a85ac24e1e3653be4a/A: appending metadata at 1733949311425 (+10 ms)Flushing mob file 786cab87062764a85ac24e1e3653be4a/A: closing flushed file at 1733949311425Flushing 786cab87062764a85ac24e1e3653be4a/A: appending metadata at 1733949311839 (+414 ms)Flushing 786cab87062764a85ac24e1e3653be4a/A: closing flushed file at 1733949311839Flushing 786cab87062764a85ac24e1e3653be4a/B: creating writer at 1733949312247 (+408 ms)Flushing 786cab87062764a85ac24e1e3653be4a/B: appending metadata at 1733949312249 (+2 ms)Flushing 786cab87062764a85ac24e1e3653be4a/B: closing flushed file at 1733949312249Flushing 786cab87062764a85ac24e1e3653be4a/C: creating writer at 1733949312663 (+414 ms)Flushing 786cab87062764a85ac24e1e3653be4a/C: appending metadata at 1733949312666 (+3 ms)Flushing 786cab87062764a85ac24e1e3653be4a/C: closing flushed file at 1733949312666Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7762a18: reopening flushed file at 1733949313075 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@513d475f: reopening flushed file at 1733949313079 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3582c96d: reopening flushed file at 1733949313084 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 786cab87062764a85ac24e1e3653be4a in 1676ms, sequenceid=430, compaction requested=true at 1733949313090 (+6 ms)Writing region close event to WAL at 1733949313129 (+39 ms)Running coprocessor post-close hooks at 1733949313133 (+4 ms)Closed at 1733949313133 2024-12-11T20:35:13,135 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,136 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=786cab87062764a85ac24e1e3653be4a, regionState=CLOSED 2024-12-11T20:35:13,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:35:13,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-11T20:35:13,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 786cab87062764a85ac24e1e3653be4a, server=5be53b084ac7,44543,1733949248752 in 1.8780 sec 2024-12-11T20:35:13,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-11T20:35:13,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=786cab87062764a85ac24e1e3653be4a, UNASSIGN in 1.8820 sec 2024-12-11T20:35:13,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-11T20:35:13,143 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8850 sec 2024-12-11T20:35:13,144 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949313144"}]},"ts":"1733949313144"} 2024-12-11T20:35:13,146 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T20:35:13,146 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T20:35:13,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8960 sec 2024-12-11T20:35:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-11T20:35:13,397 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:13,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-11T20:35:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:13,399 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:13,400 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-11T20:35:13,402 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,405 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/recovered.edits] 2024-12-11T20:35:13,409 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/73e818b44187452c9d0cec06f5a9b937 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/73e818b44187452c9d0cec06f5a9b937 2024-12-11T20:35:13,409 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/85796bd3716347e68f7cab91ffa2e3c9 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/85796bd3716347e68f7cab91ffa2e3c9 2024-12-11T20:35:13,409 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/e72478da9d1e442496fcf9207bc01b27 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/A/e72478da9d1e442496fcf9207bc01b27 2024-12-11T20:35:13,411 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/aa3e7bee47a24077a8b9681bbc742d21 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/aa3e7bee47a24077a8b9681bbc742d21 2024-12-11T20:35:13,411 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/80561e68b4f243a69326efa4f4a6e8d5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/80561e68b4f243a69326efa4f4a6e8d5 2024-12-11T20:35:13,411 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/ea9760055dde4479a88840669457cd6c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/B/ea9760055dde4479a88840669457cd6c 2024-12-11T20:35:13,414 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/32bb15c846034997b21f5e33e982d9b6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/32bb15c846034997b21f5e33e982d9b6 2024-12-11T20:35:13,414 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/764c38aab6e841b393d8f18c79e9bbf2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/764c38aab6e841b393d8f18c79e9bbf2 2024-12-11T20:35:13,414 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/57e138b97d084e36a5719ff15d3df355 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/C/57e138b97d084e36a5719ff15d3df355 2024-12-11T20:35:13,416 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/recovered.edits/433.seqid to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a/recovered.edits/433.seqid 2024-12-11T20:35:13,417 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,417 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T20:35:13,417 DEBUG [PEWorker-3 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T20:35:13,418 DEBUG [PEWorker-3 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-11T20:35:13,426 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211195bd3328cd64c49a77f088ded0843a7_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211195bd3328cd64c49a77f088ded0843a7_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,426 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412110c1bb5936916495b985695ccf61e4083_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412110c1bb5936916495b985695ccf61e4083_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,427 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121120bed42782044e4b986b6310057a8be7_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121120bed42782044e4b986b6310057a8be7_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,427 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114315442579f149eeb8176bbe93f7b7bf_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114315442579f149eeb8176bbe93f7b7bf_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,427 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412111b97697cfb2b45db8da8df359ee7e360_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412111b97697cfb2b45db8da8df359ee7e360_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,428 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211443b9e7946ba475883c6306f2686e424_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211443b9e7946ba475883c6306f2686e424_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,428 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121140f3f9bdb11b48bf8f7322ee3f31db54_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121140f3f9bdb11b48bf8f7322ee3f31db54_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,428 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121131b8e395da774d848b5a7f677b7dc0b9_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121131b8e395da774d848b5a7f677b7dc0b9_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,429 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121147e0b445daa1497abbfdd304fc2b0437_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121147e0b445daa1497abbfdd304fc2b0437_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,429 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115465867a93dd4e4da5eb5a7c471bdf54_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115465867a93dd4e4da5eb5a7c471bdf54_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,429 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211549a5a75f4df4edea96333a36d63d969_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211549a5a75f4df4edea96333a36d63d969_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,429 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121156cc32ed94314554922a6a40053d2117_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121156cc32ed94314554922a6a40053d2117_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,429 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115e1bcf1a83694a80937babe934fbc2ff_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115e1bcf1a83694a80937babe934fbc2ff_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,430 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121167d3b110573f43329b1e6d523fc7b598_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121167d3b110573f43329b1e6d523fc7b598_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,430 DEBUG [HFileArchiver-11 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117f24028f330d47c5a3150b9a366f5503_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117f24028f330d47c5a3150b9a366f5503_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,431 DEBUG [HFileArchiver-16 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211976aabe8a7d246a09476559d256e516d_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211976aabe8a7d246a09476559d256e516d_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,431 DEBUG [HFileArchiver-10 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412119888b401128b4bdb9c10093d673750cc_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412119888b401128b4bdb9c10093d673750cc_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,431 DEBUG [HFileArchiver-14 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b998f90688f94e98aa5f88c8d24478eb_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b998f90688f94e98aa5f88c8d24478eb_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,431 DEBUG [HFileArchiver-9 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bfa2e5f231304751b159618b34a46ff1_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bfa2e5f231304751b159618b34a46ff1_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,431 DEBUG [HFileArchiver-13 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d086aae291c442ac89115399c86b3a2f_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d086aae291c442ac89115399c86b3a2f_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,431 DEBUG [HFileArchiver-15 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e3d5f84d1843403d82eb4a9dea628960_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e3d5f84d1843403d82eb4a9dea628960_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,431 DEBUG [HFileArchiver-12 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f851e6dd0ed74791afa03a16f2ef44a4_786cab87062764a85ac24e1e3653be4a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211f851e6dd0ed74791afa03a16f2ef44a4_786cab87062764a85ac24e1e3653be4a 2024-12-11T20:35:13,432 DEBUG [PEWorker-3 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T20:35:13,434 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:13,437 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T20:35:13,439 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T20:35:13,440 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:13,440 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T20:35:13,440 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733949313440"}]},"ts":"9223372036854775807"} 2024-12-11T20:35:13,442 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T20:35:13,442 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 786cab87062764a85ac24e1e3653be4a, NAME => 'TestAcidGuarantees,,1733949280528.786cab87062764a85ac24e1e3653be4a.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T20:35:13,442 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T20:35:13,443 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733949313442"}]},"ts":"9223372036854775807"} 2024-12-11T20:35:13,445 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T20:35:13,446 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:13,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-12-11T20:35:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-11T20:35:13,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T20:35:13,507 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:13,516 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=255 (was 257), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=484 (was 400) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1890 (was 2164) 2024-12-11T20:35:13,525 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=255, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=484, ProcessCount=11, AvailableMemoryMB=1890 2024-12-11T20:35:13,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T20:35:13,526 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:35:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:13,528 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:35:13,528 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:13,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-12-11T20:35:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-11T20:35:13,529 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:35:13,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742069_1245 (size=1061) 2024-12-11T20:35:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-11T20:35:13,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-11T20:35:13,936 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043 2024-12-11T20:35:13,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742070_1246 (size=53) 2024-12-11T20:35:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-11T20:35:14,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:35:14,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 320c74933b0c438c8808bab076a15dce, disabling compactions & flushes 2024-12-11T20:35:14,342 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:14,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:14,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. after waiting 0 ms 2024-12-11T20:35:14,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:14,342 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:14,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 320c74933b0c438c8808bab076a15dce: Waiting for close lock at 1733949314342Disabling compacts and flushes for region at 1733949314342Disabling writes for close at 1733949314342Writing region close event to WAL at 1733949314342Closed at 1733949314342 2024-12-11T20:35:14,343 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:35:14,343 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733949314343"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733949314343"}]},"ts":"1733949314343"} 2024-12-11T20:35:14,345 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:35:14,346 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:35:14,346 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949314346"}]},"ts":"1733949314346"} 2024-12-11T20:35:14,347 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T20:35:14,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=320c74933b0c438c8808bab076a15dce, ASSIGN}] 2024-12-11T20:35:14,348 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=320c74933b0c438c8808bab076a15dce, ASSIGN 2024-12-11T20:35:14,349 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=320c74933b0c438c8808bab076a15dce, ASSIGN; state=OFFLINE, location=5be53b084ac7,44543,1733949248752; forceNewPlan=false, retain=false 2024-12-11T20:35:14,500 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=320c74933b0c438c8808bab076a15dce, regionState=OPENING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:14,501 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=320c74933b0c438c8808bab076a15dce, ASSIGN because future has completed 2024-12-11T20:35:14,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:35:14,657 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:14,657 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:35:14,657 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,657 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:35:14,658 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,658 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,659 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,660 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:35:14,660 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 320c74933b0c438c8808bab076a15dce columnFamilyName A 2024-12-11T20:35:14,660 DEBUG [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:14,661 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] regionserver.HStore(327): Store=320c74933b0c438c8808bab076a15dce/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:35:14,661 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,662 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:35:14,662 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 320c74933b0c438c8808bab076a15dce columnFamilyName B 2024-12-11T20:35:14,662 DEBUG [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:14,662 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] regionserver.HStore(327): Store=320c74933b0c438c8808bab076a15dce/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:35:14,662 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,663 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:35:14,663 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 320c74933b0c438c8808bab076a15dce columnFamilyName C 2024-12-11T20:35:14,663 DEBUG [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:14,664 INFO [StoreOpener-320c74933b0c438c8808bab076a15dce-1 {}] regionserver.HStore(327): Store=320c74933b0c438c8808bab076a15dce/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:35:14,664 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:14,664 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,665 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,665 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,666 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,666 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-11T20:35:14,667 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T20:35:14,668 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,669 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:35:14,670 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 320c74933b0c438c8808bab076a15dce; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69633379, jitterRate=0.037618204951286316}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T20:35:14,670 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:14,670 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 320c74933b0c438c8808bab076a15dce: Running coprocessor pre-open hook at 1733949314658Writing region info on filesystem at 1733949314658Initializing all the Stores at 1733949314659 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949314659Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949314659Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949314659Cleaning up temporary data from old regions at 1733949314666 (+7 ms)Running coprocessor post-open hooks at 1733949314670 (+4 ms)Region opened successfully at 1733949314670 2024-12-11T20:35:14,671 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., pid=65, masterSystemTime=1733949314654 2024-12-11T20:35:14,673 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:14,673 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:14,673 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=320c74933b0c438c8808bab076a15dce, regionState=OPEN, openSeqNum=2, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:14,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:35:14,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-11T20:35:14,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 in 173 msec 2024-12-11T20:35:14,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-11T20:35:14,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=320c74933b0c438c8808bab076a15dce, ASSIGN in 330 msec 2024-12-11T20:35:14,679 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:35:14,679 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949314679"}]},"ts":"1733949314679"} 2024-12-11T20:35:14,680 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T20:35:14,681 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:35:14,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-12-11T20:35:15,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-11T20:35:15,677 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:15,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626eaa43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:15,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:15,679 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3489eea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:15,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:15,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:15,681 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:15,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b296778, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,682 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:15,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:15,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:15,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:15,684 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T20:35:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 5be53b084ac7,39923,1733949248041 2024-12-11T20:35:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2e2b076a 2024-12-11T20:35:15,685 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T20:35:15,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T20:35:15,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d6abe12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:15,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:15,688 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:15,688 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ddbf08e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:15,689 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@757f9370, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37499dc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:15,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:15,692 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52812d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:15,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:15,693 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:15,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7f906c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a30a513, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:15,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:15,695 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:15,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302aae5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:15,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:15,696 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:15,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1a3613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,699 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753be273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,699 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:15,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:15,700 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74402d7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:15,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:15,702 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:15,702 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49470cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,703 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12cb2878, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,703 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:15,703 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:15,704 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:15,704 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a88b469, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:15,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:15,706 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@177f3110, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d293888, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:15,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:15,708 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:15,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:15,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:15,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195a68f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:15,709 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:15,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd5d3ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19043b9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,710 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:15,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:15,711 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:15,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27aac594, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:15,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:15,713 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54702, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@372d2110, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@516f54e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:15,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:15,715 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:15,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:15,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:15,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2089b81a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:15,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:15,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:15,716 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54714, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:15,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8b88f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4287b3a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:15,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:15,718 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:15,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:15,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62d20022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:15,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:15,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a3c7ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5239e036, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:15,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:15,724 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@734ce248, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:15,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:15,725 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:15,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b760d46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:15,727 DEBUG [Thread-1147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:15,727 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:15,727 DEBUG [Thread-1145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:15,727 DEBUG [Thread-1156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:15,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:15,727 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:15,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:15,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:15,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:15,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:15,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:15,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:15,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:15,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:15,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:15,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:15,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:35:15,732 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:35:15,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:35:15,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:35:15,740 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:15,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-11T20:35:15,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:15,743 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:15,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T20:35:15,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T20:35:15,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:15,744 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:15,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:15,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:15,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:15,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:15,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:15,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:15,747 DEBUG [Thread-1160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:15,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:15,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:15,753 DEBUG [Thread-1162 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:15,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:35:15,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:15,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:35:15,767 DEBUG [Thread-1151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:15,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:15,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:35:15,779 DEBUG [Thread-1143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:15,780 DEBUG [Thread-1143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:15,780 DEBUG [Thread-1143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:15,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:15,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:35:15,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/3fd1bff5b9fc4917a40f835ca2ed0dfc is 50, key is test_row_0/A:col10/1733949315737/Put/seqid=0 2024-12-11T20:35:15,787 DEBUG [Thread-1158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:15,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:15,789 DEBUG [Thread-1149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:15,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:15,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:35:15,792 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:15,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:35:15,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742071_1247 (size=12001) 2024-12-11T20:35:15,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/3fd1bff5b9fc4917a40f835ca2ed0dfc 2024-12-11T20:35:15,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949315764, max timestamp=1733949315813Num uniques 859; 2024-12-11T20:35:15,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:15,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733949315764, max timestamp=1733949315813Num uniques 859; 2024-12-11T20:35:15,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-11T20:35:15,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733949315764, max timestamp=1733949315813Num uniques 859; 2024-12-11T20:35:15,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-11T20:35:15,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/c06cd746318b497b8d4c896a6deaeaf8 is 50, key is test_row_0/B:col10/1733949315737/Put/seqid=0 2024-12-11T20:35:15,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742072_1248 (size=12001) 2024-12-11T20:35:15,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/c06cd746318b497b8d4c896a6deaeaf8 2024-12-11T20:35:15,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49322 deadline: 1733949375847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:15,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49266 deadline: 1733949375848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:15,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:15,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949375849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:15,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T20:35:15,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:15,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49298 deadline: 1733949375855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:15,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:15,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949375856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:15,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:15,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/3e25923833d0445583ddba3fee3d41e4 is 50, key is test_row_0/C:col10/1733949315737/Put/seqid=0 2024-12-11T20:35:15,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T20:35:15,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:15,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:15,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:15,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742073_1249 (size=12001) 2024-12-11T20:35:15,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/3e25923833d0445583ddba3fee3d41e4 2024-12-11T20:35:15,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/3fd1bff5b9fc4917a40f835ca2ed0dfc as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/3fd1bff5b9fc4917a40f835ca2ed0dfc 2024-12-11T20:35:15,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/3fd1bff5b9fc4917a40f835ca2ed0dfc, entries=150, sequenceid=12, filesize=11.7 K 2024-12-11T20:35:15,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/c06cd746318b497b8d4c896a6deaeaf8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/c06cd746318b497b8d4c896a6deaeaf8 2024-12-11T20:35:15,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/c06cd746318b497b8d4c896a6deaeaf8, entries=150, sequenceid=12, filesize=11.7 K 2024-12-11T20:35:15,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/3e25923833d0445583ddba3fee3d41e4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3e25923833d0445583ddba3fee3d41e4 2024-12-11T20:35:15,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3e25923833d0445583ddba3fee3d41e4, entries=150, sequenceid=12, filesize=11.7 K 2024-12-11T20:35:15,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=133.39 KB/136590 for 320c74933b0c438c8808bab076a15dce in 196ms, sequenceid=12, compaction requested=false 2024-12-11T20:35:15,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:15,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949315845, max timestamp=1733949315968Num uniques 1009; 2024-12-11T20:35:15,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949315845, max timestamp=1733949315968Num uniques 1018; 2024-12-11T20:35:15,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T20:35:15,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-11T20:35:15,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949315845, max timestamp=1733949315968Num uniques 1009; 2024-12-11T20:35:15,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-11T20:35:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:15,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=53.67 KB heapSize=145.08 KB 2024-12-11T20:35:15,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:15,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:15,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:15,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:15,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:15,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:15,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/c5136573d6e8464f81eddbd088cfa486 is 50, key is test_row_0/A:col10/1733949315972/Put/seqid=0 2024-12-11T20:35:15,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742074_1250 (size=12001) 2024-12-11T20:35:15,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/c5136573d6e8464f81eddbd088cfa486 2024-12-11T20:35:16,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/39930a7752f44dc98199c41efbfa201e is 50, key is test_row_0/B:col10/1733949315972/Put/seqid=0 2024-12-11T20:35:16,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949376020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49298 deadline: 1733949376023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949376025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49322 deadline: 1733949376026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49266 deadline: 1733949376026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742075_1251 (size=12001) 2024-12-11T20:35:16,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/39930a7752f44dc98199c41efbfa201e 2024-12-11T20:35:16,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/55c1df3968794321aa664fe7ac314e76 is 50, key is test_row_0/C:col10/1733949315972/Put/seqid=0 2024-12-11T20:35:16,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T20:35:16,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:16,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T20:35:16,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742076_1252 (size=12001) 2024-12-11T20:35:16,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/55c1df3968794321aa664fe7ac314e76 2024-12-11T20:35:16,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/c5136573d6e8464f81eddbd088cfa486 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/c5136573d6e8464f81eddbd088cfa486 2024-12-11T20:35:16,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/c5136573d6e8464f81eddbd088cfa486, entries=150, sequenceid=56, filesize=11.7 K 2024-12-11T20:35:16,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/39930a7752f44dc98199c41efbfa201e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/39930a7752f44dc98199c41efbfa201e 2024-12-11T20:35:16,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/39930a7752f44dc98199c41efbfa201e, entries=150, sequenceid=56, filesize=11.7 K 2024-12-11T20:35:16,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/55c1df3968794321aa664fe7ac314e76 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/55c1df3968794321aa664fe7ac314e76 2024-12-11T20:35:16,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/55c1df3968794321aa664fe7ac314e76, entries=150, sequenceid=56, filesize=11.7 K 2024-12-11T20:35:16,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~184.88 KB/189312, currentSize=114.05 KB/116790 for 320c74933b0c438c8808bab076a15dce in 141ms, sequenceid=56, compaction requested=false 2024-12-11T20:35:16,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:16,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949316020, max timestamp=1733949316139Num uniques 859; 2024-12-11T20:35:16,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:16,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-11T20:35:16,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:16,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949316020, max timestamp=1733949316139Num uniques 859; 2024-12-11T20:35:16,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:16,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949316020, max timestamp=1733949316139Num uniques 859; 2024-12-11T20:35:16,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:16,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:16,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:16,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:16,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:16,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:16,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:16,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/1075166efd8b47ac9106823bbbba550f is 50, key is test_row_0/A:col10/1733949316140/Put/seqid=0 2024-12-11T20:35:16,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742077_1253 (size=12001) 2024-12-11T20:35:16,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/1075166efd8b47ac9106823bbbba550f 2024-12-11T20:35:16,195 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733949316173, max timestamp=1733949316192Num uniques 859; 2024-12-11T20:35:16,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-11T20:35:16,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949316187, max timestamp=1733949316192Num uniques 859; 2024-12-11T20:35:16,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:16,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949316187, max timestamp=1733949316192Num uniques 859; 2024-12-11T20:35:16,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:16,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/351a7d08f9624969918dad623f5cdf18 is 50, key is test_row_0/B:col10/1733949316140/Put/seqid=0 2024-12-11T20:35:16,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T20:35:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:16,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742078_1254 (size=12001) 2024-12-11T20:35:16,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949376223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949376224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,231 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733949316214, max timestamp=1733949316229Num uniques 1168; 2024-12-11T20:35:16,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=53488, new segment data size=9978, suffix heap size=149440,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1168, new segment cells count=218 2024-12-11T20:35:16,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733949316214, max timestamp=1733949316229Num uniques 1018; 2024-12-11T20:35:16,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-11T20:35:16,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733949316214, max timestamp=1733949316229Num uniques 1018; 2024-12-11T20:35:16,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-11T20:35:16,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10382, totalHeapSize=23848, min timestamp=1733949316251, max timestamp=1733949316274Num uniques 1077; 2024-12-11T20:35:16,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10382, totalHeapSize=23848, min timestamp=1733949316251, max timestamp=1733949316274Num uniques 1077; 2024-12-11T20:35:16,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49312, new segment data size=10382, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-11T20:35:16,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49312, new segment data size=10382, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-11T20:35:16,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10382, totalHeapSize=23848, min timestamp=1733949316251, max timestamp=1733949316274Num uniques 1077; 2024-12-11T20:35:16,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49312, new segment data size=10382, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-12-11T20:35:16,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949376336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733949316314, max timestamp=1733949316346Num uniques 1086; 2024-12-11T20:35:16,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-11T20:35:16,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733949316314, max timestamp=1733949316346Num uniques 1086; 2024-12-11T20:35:16,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-11T20:35:16,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1733949316314, max timestamp=1733949316346Num uniques 1086; 2024-12-11T20:35:16,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-12-11T20:35:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T20:35:16,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:16,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T20:35:16,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733949316364, max timestamp=1733949316377Num uniques 1145; 2024-12-11T20:35:16,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733949316364, max timestamp=1733949316377Num uniques 1145; 2024-12-11T20:35:16,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733949316364, max timestamp=1733949316377Num uniques 1145; 2024-12-11T20:35:16,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-11T20:35:16,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-11T20:35:16,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=13510, suffix heap size=147048,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=295 2024-12-11T20:35:16,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49298 deadline: 1733949376408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49322 deadline: 1733949376408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949376410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49266 deadline: 1733949376411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,415 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T20:35:16,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:16,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49298 deadline: 1733949376516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949376516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49266 deadline: 1733949376517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49322 deadline: 1733949376517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949376557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/351a7d08f9624969918dad623f5cdf18 2024-12-11T20:35:16,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/02354fdd56e04eabbbaa0c6cb7c1326d is 50, key is test_row_0/C:col10/1733949316140/Put/seqid=0 2024-12-11T20:35:16,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742079_1255 (size=12001) 2024-12-11T20:35:16,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T20:35:16,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:16,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49298 deadline: 1733949376726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49322 deadline: 1733949376727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49266 deadline: 1733949376727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949376727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,731 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T20:35:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:16,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:16,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949376867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:16,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:16,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T20:35:16,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T20:35:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:16,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:17,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/02354fdd56e04eabbbaa0c6cb7c1326d 2024-12-11T20:35:17,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/1075166efd8b47ac9106823bbbba550f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1075166efd8b47ac9106823bbbba550f 2024-12-11T20:35:17,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1075166efd8b47ac9106823bbbba550f, entries=150, sequenceid=79, filesize=11.7 K 2024-12-11T20:35:17,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/351a7d08f9624969918dad623f5cdf18 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/351a7d08f9624969918dad623f5cdf18 2024-12-11T20:35:17,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/351a7d08f9624969918dad623f5cdf18, entries=150, sequenceid=79, filesize=11.7 K 2024-12-11T20:35:17,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:17,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49322 deadline: 1733949377037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:17,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/02354fdd56e04eabbbaa0c6cb7c1326d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/02354fdd56e04eabbbaa0c6cb7c1326d 2024-12-11T20:35:17,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49298 deadline: 1733949377037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:17,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949377037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,040 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:17,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49266 deadline: 1733949377037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:17,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:17,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/02354fdd56e04eabbbaa0c6cb7c1326d, entries=150, sequenceid=79, filesize=11.7 K 2024-12-11T20:35:17,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=147.60 KB/151140 for 320c74933b0c438c8808bab076a15dce in 902ms, sequenceid=79, compaction requested=true 2024-12-11T20:35:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:17,045 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:17,045 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:17,046 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:17,046 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:17,046 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/B is initiating minor compaction (all files) 2024-12-11T20:35:17,046 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/A is initiating minor compaction (all files) 2024-12-11T20:35:17,047 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/B in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:17,047 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/A in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:17,047 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/3fd1bff5b9fc4917a40f835ca2ed0dfc, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/c5136573d6e8464f81eddbd088cfa486, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1075166efd8b47ac9106823bbbba550f] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=35.2 K 2024-12-11T20:35:17,047 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/c06cd746318b497b8d4c896a6deaeaf8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/39930a7752f44dc98199c41efbfa201e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/351a7d08f9624969918dad623f5cdf18] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=35.2 K 2024-12-11T20:35:17,047 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fd1bff5b9fc4917a40f835ca2ed0dfc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733949315737 2024-12-11T20:35:17,047 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c06cd746318b497b8d4c896a6deaeaf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733949315737 2024-12-11T20:35:17,047 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5136573d6e8464f81eddbd088cfa486, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733949315972 2024-12-11T20:35:17,048 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 39930a7752f44dc98199c41efbfa201e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733949315972 2024-12-11T20:35:17,048 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1075166efd8b47ac9106823bbbba550f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733949316139 2024-12-11T20:35:17,048 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 351a7d08f9624969918dad623f5cdf18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733949316139 2024-12-11T20:35:17,069 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#B#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:17,070 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/0ff2e44d05814e3dbdb785e1883a768d is 50, key is test_row_0/B:col10/1733949316140/Put/seqid=0 2024-12-11T20:35:17,072 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:17,072 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/567eee4c67a74d4d993d72e10984c248 is 50, key is test_row_0/A:col10/1733949316140/Put/seqid=0 2024-12-11T20:35:17,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742080_1256 (size=12104) 2024-12-11T20:35:17,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742081_1257 (size=12104) 2024-12-11T20:35:17,085 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/567eee4c67a74d4d993d72e10984c248 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/567eee4c67a74d4d993d72e10984c248 2024-12-11T20:35:17,090 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/A of 320c74933b0c438c8808bab076a15dce into 567eee4c67a74d4d993d72e10984c248(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:17,090 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:17,090 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/A, priority=13, startTime=1733949317045; duration=0sec 2024-12-11T20:35:17,090 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:17,090 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:A 2024-12-11T20:35:17,090 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:17,092 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:17,092 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/C is initiating minor compaction (all files) 2024-12-11T20:35:17,092 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/C in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:17,092 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3e25923833d0445583ddba3fee3d41e4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/55c1df3968794321aa664fe7ac314e76, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/02354fdd56e04eabbbaa0c6cb7c1326d] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=35.2 K 2024-12-11T20:35:17,092 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e25923833d0445583ddba3fee3d41e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733949315737 2024-12-11T20:35:17,100 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55c1df3968794321aa664fe7ac314e76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1733949315972 2024-12-11T20:35:17,101 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02354fdd56e04eabbbaa0c6cb7c1326d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733949316139 2024-12-11T20:35:17,108 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#C#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:17,109 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/3be7e62335e34effa39a1312f14204d5 is 50, key is test_row_0/C:col10/1733949316140/Put/seqid=0 2024-12-11T20:35:17,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-11T20:35:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:17,125 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=147.60 KB heapSize=412.08 KB 2024-12-11T20:35:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:17,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:17,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742082_1258 (size=12104) 2024-12-11T20:35:17,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/e3bb486bd0584124bb1f11081d23af84 is 50, key is test_row_0/A:col10/1733949316409/Put/seqid=0 2024-12-11T20:35:17,134 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/3be7e62335e34effa39a1312f14204d5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3be7e62335e34effa39a1312f14204d5 2024-12-11T20:35:17,140 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/C of 320c74933b0c438c8808bab076a15dce into 3be7e62335e34effa39a1312f14204d5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:17,140 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:17,140 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/C, priority=13, startTime=1733949317045; duration=0sec 2024-12-11T20:35:17,140 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:17,140 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:C 2024-12-11T20:35:17,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742083_1259 (size=12151) 2024-12-11T20:35:17,146 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/e3bb486bd0584124bb1f11081d23af84 2024-12-11T20:35:17,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/dd2e24a4029e4eae874dbc97d9337c02 is 50, key is test_row_0/B:col10/1733949316409/Put/seqid=0 2024-12-11T20:35:17,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742084_1260 (size=12151) 2024-12-11T20:35:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:17,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:17,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949377409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:17,482 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/0ff2e44d05814e3dbdb785e1883a768d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/0ff2e44d05814e3dbdb785e1883a768d 2024-12-11T20:35:17,488 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/B of 320c74933b0c438c8808bab076a15dce into 0ff2e44d05814e3dbdb785e1883a768d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:17,488 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:17,488 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/B, priority=13, startTime=1733949317045; duration=0sec 2024-12-11T20:35:17,489 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:17,489 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:B 2024-12-11T20:35:17,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:17,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949377516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:17,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49322 deadline: 1733949377547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:17,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:17,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49298 deadline: 1733949377547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:17,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:17,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949377547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:17,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:17,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49266 deadline: 1733949377547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:17,562 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/dd2e24a4029e4eae874dbc97d9337c02 2024-12-11T20:35:17,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/d2267dcb1381474fba933529ea01fb21 is 50, key is test_row_0/C:col10/1733949316409/Put/seqid=0 2024-12-11T20:35:17,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742085_1261 (size=12151) 2024-12-11T20:35:17,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949377726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:17,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:17,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T20:35:18,002 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/d2267dcb1381474fba933529ea01fb21 2024-12-11T20:35:18,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/e3bb486bd0584124bb1f11081d23af84 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e3bb486bd0584124bb1f11081d23af84 2024-12-11T20:35:18,011 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e3bb486bd0584124bb1f11081d23af84, entries=150, sequenceid=185, filesize=11.9 K 2024-12-11T20:35:18,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/dd2e24a4029e4eae874dbc97d9337c02 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/dd2e24a4029e4eae874dbc97d9337c02 2024-12-11T20:35:18,018 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/dd2e24a4029e4eae874dbc97d9337c02, entries=150, sequenceid=185, filesize=11.9 K 2024-12-11T20:35:18,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/d2267dcb1381474fba933529ea01fb21 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/d2267dcb1381474fba933529ea01fb21 2024-12-11T20:35:18,026 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/d2267dcb1381474fba933529ea01fb21, entries=150, sequenceid=185, filesize=11.9 K 2024-12-11T20:35:18,027 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~412.03 KB/421920, currentSize=33.54 KB/34350 for 320c74933b0c438c8808bab076a15dce in 902ms, sequenceid=185, compaction requested=false 2024-12-11T20:35:18,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:18,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:18,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-11T20:35:18,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-11T20:35:18,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-11T20:35:18,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2840 sec 2024-12-11T20:35:18,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.2910 sec 2024-12-11T20:35:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:18,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:35:18,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:18,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:18,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:18,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:18,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:18,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:18,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/7e2229a133454d048fefa9eddbf1e7a5 is 50, key is test_row_0/A:col10/1733949318040/Put/seqid=0 2024-12-11T20:35:18,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742086_1262 (size=14541) 2024-12-11T20:35:18,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949318090, max timestamp=1733949318110Num uniques 859; 2024-12-11T20:35:18,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:18,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949318090, max timestamp=1733949318110Num uniques 859; 2024-12-11T20:35:18,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:18,112 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949318090, max timestamp=1733949318110Num uniques 859; 2024-12-11T20:35:18,113 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:18,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949378194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:18,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949378307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:18,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/7e2229a133454d048fefa9eddbf1e7a5 2024-12-11T20:35:18,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/cc231006ab1d4397a049ea9b708d0f71 is 50, key is test_row_0/B:col10/1733949318040/Put/seqid=0 2024-12-11T20:35:18,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742087_1263 (size=12151) 2024-12-11T20:35:18,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949378517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:18,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:18,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49322 deadline: 1733949378556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:18,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:18,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49266 deadline: 1733949378556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:18,558 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:18,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:18,559 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:18,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,559 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:18,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:18,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949378567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:18,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:18,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49298 deadline: 1733949378567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:18,570 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:18,570 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_0' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:18,660 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:35:18,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949378827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:18,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:18,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/cc231006ab1d4397a049ea9b708d0f71 2024-12-11T20:35:18,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/f2a03473fb0f4ed6a3189db83378e272 is 50, key is test_row_0/C:col10/1733949318040/Put/seqid=0 2024-12-11T20:35:18,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742088_1264 (size=12151) 2024-12-11T20:35:19,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/f2a03473fb0f4ed6a3189db83378e272 2024-12-11T20:35:19,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/7e2229a133454d048fefa9eddbf1e7a5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/7e2229a133454d048fefa9eddbf1e7a5 2024-12-11T20:35:19,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/7e2229a133454d048fefa9eddbf1e7a5, entries=200, sequenceid=197, filesize=14.2 K 2024-12-11T20:35:19,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/cc231006ab1d4397a049ea9b708d0f71 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/cc231006ab1d4397a049ea9b708d0f71 2024-12-11T20:35:19,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/cc231006ab1d4397a049ea9b708d0f71, entries=150, sequenceid=197, filesize=11.9 K 2024-12-11T20:35:19,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/f2a03473fb0f4ed6a3189db83378e272 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/f2a03473fb0f4ed6a3189db83378e272 2024-12-11T20:35:19,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/f2a03473fb0f4ed6a3189db83378e272, entries=150, sequenceid=197, filesize=11.9 K 2024-12-11T20:35:19,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 320c74933b0c438c8808bab076a15dce in 1260ms, sequenceid=197, compaction requested=true 2024-12-11T20:35:19,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:19,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:19,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:19,302 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:19,302 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:19,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:19,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:19,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:19,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:19,303 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38796 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:19,303 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:19,303 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/B is initiating minor compaction (all files) 2024-12-11T20:35:19,303 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/A is initiating minor compaction (all files) 2024-12-11T20:35:19,303 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/B in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:19,303 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/A in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:19,303 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/567eee4c67a74d4d993d72e10984c248, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e3bb486bd0584124bb1f11081d23af84, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/7e2229a133454d048fefa9eddbf1e7a5] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=37.9 K 2024-12-11T20:35:19,303 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/0ff2e44d05814e3dbdb785e1883a768d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/dd2e24a4029e4eae874dbc97d9337c02, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/cc231006ab1d4397a049ea9b708d0f71] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=35.6 K 2024-12-11T20:35:19,304 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 567eee4c67a74d4d993d72e10984c248, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733949316139 2024-12-11T20:35:19,304 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ff2e44d05814e3dbdb785e1883a768d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733949316139 2024-12-11T20:35:19,304 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting dd2e24a4029e4eae874dbc97d9337c02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733949316398 2024-12-11T20:35:19,304 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3bb486bd0584124bb1f11081d23af84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733949316398 2024-12-11T20:35:19,305 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting cc231006ab1d4397a049ea9b708d0f71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733949317392 2024-12-11T20:35:19,305 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e2229a133454d048fefa9eddbf1e7a5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733949317392 2024-12-11T20:35:19,314 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#B#compaction#218 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:19,315 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/2776acf147d2441890e963c7bfa97c8a is 50, key is test_row_0/B:col10/1733949318040/Put/seqid=0 2024-12-11T20:35:19,321 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#A#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:19,322 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/9cbcb387c85d4505994b7ac2451becaa is 50, key is test_row_0/A:col10/1733949318040/Put/seqid=0 2024-12-11T20:35:19,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-11T20:35:19,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:19,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T20:35:19,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:19,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T20:35:19,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:19,341 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T20:35:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:19,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/877735eddc754244a3b5263171a4ee9c is 50, key is test_row_0/A:col10/1733949318188/Put/seqid=0 2024-12-11T20:35:19,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742089_1265 (size=12357) 2024-12-11T20:35:19,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742090_1266 (size=12357) 2024-12-11T20:35:19,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742091_1267 (size=12151) 2024-12-11T20:35:19,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:19,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949379369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:19,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:19,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:19,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949379477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:19,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:19,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949379686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:19,757 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/2776acf147d2441890e963c7bfa97c8a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/2776acf147d2441890e963c7bfa97c8a 2024-12-11T20:35:19,763 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/B of 320c74933b0c438c8808bab076a15dce into 2776acf147d2441890e963c7bfa97c8a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:19,763 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:19,763 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/B, priority=13, startTime=1733949319302; duration=0sec 2024-12-11T20:35:19,764 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:19,764 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:B 2024-12-11T20:35:19,764 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:19,765 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:19,765 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/C is initiating minor compaction (all files) 2024-12-11T20:35:19,765 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/C in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:19,765 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3be7e62335e34effa39a1312f14204d5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/d2267dcb1381474fba933529ea01fb21, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/f2a03473fb0f4ed6a3189db83378e272] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=35.6 K 2024-12-11T20:35:19,765 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3be7e62335e34effa39a1312f14204d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1733949316139 2024-12-11T20:35:19,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/877735eddc754244a3b5263171a4ee9c 2024-12-11T20:35:19,766 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d2267dcb1381474fba933529ea01fb21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1733949316398 2024-12-11T20:35:19,767 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f2a03473fb0f4ed6a3189db83378e272, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733949317392 2024-12-11T20:35:19,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/f9a49dbfe18d476696f478c3f2351d38 is 50, key is test_row_0/B:col10/1733949318188/Put/seqid=0 2024-12-11T20:35:19,779 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/9cbcb387c85d4505994b7ac2451becaa as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/9cbcb387c85d4505994b7ac2451becaa 2024-12-11T20:35:19,780 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#C#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:19,781 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/7984b6d344e547d181a96d4943e6336b is 50, key is test_row_0/C:col10/1733949318040/Put/seqid=0 2024-12-11T20:35:19,787 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/A of 320c74933b0c438c8808bab076a15dce into 9cbcb387c85d4505994b7ac2451becaa(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:19,787 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:19,787 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/A, priority=13, startTime=1733949319302; duration=0sec 2024-12-11T20:35:19,788 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:19,788 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:A 2024-12-11T20:35:19,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742093_1269 (size=12151) 2024-12-11T20:35:19,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742092_1268 (size=12357) 2024-12-11T20:35:19,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-11T20:35:19,917 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:19,918 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-11T20:35:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-11T20:35:19,920 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:19,920 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:19,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:19,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:19,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949379997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:19,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-11T20:35:20,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-11T20:35:20,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:20,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:20,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:20,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/f9a49dbfe18d476696f478c3f2351d38 2024-12-11T20:35:20,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/1101f4f214e74cfc95fab108f209709d is 50, key is test_row_0/C:col10/1733949318188/Put/seqid=0 2024-12-11T20:35:20,201 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/7984b6d344e547d181a96d4943e6336b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/7984b6d344e547d181a96d4943e6336b 2024-12-11T20:35:20,207 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/C of 320c74933b0c438c8808bab076a15dce into 7984b6d344e547d181a96d4943e6336b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:20,207 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:20,207 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/C, priority=13, startTime=1733949319302; duration=0sec 2024-12-11T20:35:20,207 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:20,207 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:C 2024-12-11T20:35:20,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742094_1270 (size=12151) 2024-12-11T20:35:20,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-11T20:35:20,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:20,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:20,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:20,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-11T20:35:20,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-11T20:35:20,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:20,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:20,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:20,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:20,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949380507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:20,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:20,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-11T20:35:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:20,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:20,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:20,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-11T20:35:20,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49266 deadline: 1733949380566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:20,567 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:20,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49322 deadline: 1733949380586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:20,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949380587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:20,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:20,589 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49298 deadline: 1733949380587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:20,590 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:20,590 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_0' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:20,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:20,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/1101f4f214e74cfc95fab108f209709d 2024-12-11T20:35:20,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/877735eddc754244a3b5263171a4ee9c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/877735eddc754244a3b5263171a4ee9c 2024-12-11T20:35:20,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/877735eddc754244a3b5263171a4ee9c, entries=150, sequenceid=235, filesize=11.9 K 2024-12-11T20:35:20,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/f9a49dbfe18d476696f478c3f2351d38 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f9a49dbfe18d476696f478c3f2351d38 2024-12-11T20:35:20,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f9a49dbfe18d476696f478c3f2351d38, entries=150, sequenceid=235, filesize=11.9 K 2024-12-11T20:35:20,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/1101f4f214e74cfc95fab108f209709d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/1101f4f214e74cfc95fab108f209709d 2024-12-11T20:35:20,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/1101f4f214e74cfc95fab108f209709d, entries=150, sequenceid=235, filesize=11.9 K 2024-12-11T20:35:20,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 320c74933b0c438c8808bab076a15dce in 1298ms, sequenceid=235, compaction requested=false 2024-12-11T20:35:20,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:20,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-11T20:35:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:20,685 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-11T20:35:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:20,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:20,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/1d33f08788514f8189719a224cd493d1 is 50, key is test_row_0/A:col10/1733949319358/Put/seqid=0 2024-12-11T20:35:20,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742095_1271 (size=12151) 2024-12-11T20:35:20,694 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/1d33f08788514f8189719a224cd493d1 2024-12-11T20:35:20,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/1275246dfd0742fc938de721ae15b415 is 50, key is test_row_0/B:col10/1733949319358/Put/seqid=0 2024-12-11T20:35:20,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742096_1272 (size=12151) 2024-12-11T20:35:21,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-11T20:35:21,110 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/1275246dfd0742fc938de721ae15b415 2024-12-11T20:35:21,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/4974f89c5ec24627b212176eb42e355b is 50, key is test_row_0/C:col10/1733949319358/Put/seqid=0 2024-12-11T20:35:21,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742097_1273 (size=12151) 2024-12-11T20:35:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:21,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:21,531 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/4974f89c5ec24627b212176eb42e355b 2024-12-11T20:35:21,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/1d33f08788514f8189719a224cd493d1 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1d33f08788514f8189719a224cd493d1 2024-12-11T20:35:21,541 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1d33f08788514f8189719a224cd493d1, entries=150, sequenceid=247, filesize=11.9 K 2024-12-11T20:35:21,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/1275246dfd0742fc938de721ae15b415 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1275246dfd0742fc938de721ae15b415 2024-12-11T20:35:21,547 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1275246dfd0742fc938de721ae15b415, entries=150, sequenceid=247, filesize=11.9 K 2024-12-11T20:35:21,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/4974f89c5ec24627b212176eb42e355b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/4974f89c5ec24627b212176eb42e355b 2024-12-11T20:35:21,553 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/4974f89c5ec24627b212176eb42e355b, entries=150, sequenceid=247, filesize=11.9 K 2024-12-11T20:35:21,553 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=67.09 KB/68700 for 320c74933b0c438c8808bab076a15dce in 868ms, sequenceid=247, compaction requested=true 2024-12-11T20:35:21,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-11T20:35:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-11T20:35:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:21,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-11T20:35:21,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:21,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:21,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:21,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:21,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:21,556 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:21,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-11T20:35:21,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6340 sec 2024-12-11T20:35:21,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6400 sec 2024-12-11T20:35:21,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/61a8917ed21e426394bc9e9bbc6f05a0 is 50, key is test_row_0/A:col10/1733949321549/Put/seqid=0 2024-12-11T20:35:21,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742098_1274 (size=14691) 2024-12-11T20:35:21,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949381618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:21,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:21,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949381726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:21,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:21,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949381937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:21,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/61a8917ed21e426394bc9e9bbc6f05a0 2024-12-11T20:35:21,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/d600d020c8e04b5782ece5d65bdc158c is 50, key is test_row_0/B:col10/1733949321549/Put/seqid=0 2024-12-11T20:35:21,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742099_1275 (size=12251) 2024-12-11T20:35:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-11T20:35:22,067 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:22,067 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-11T20:35:22,069 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:22,070 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:22,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T20:35:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T20:35:22,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T20:35:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949382246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:22,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T20:35:22,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:22,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/d600d020c8e04b5782ece5d65bdc158c 2024-12-11T20:35:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T20:35:22,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/9841c362fbd145ebb2d85b515e35986e is 50, key is test_row_0/C:col10/1733949321549/Put/seqid=0 2024-12-11T20:35:22,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742100_1276 (size=12251) 2024-12-11T20:35:22,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T20:35:22,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:22,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T20:35:22,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:22,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:22,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T20:35:22,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:22,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949382757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:22,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/9841c362fbd145ebb2d85b515e35986e 2024-12-11T20:35:22,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/61a8917ed21e426394bc9e9bbc6f05a0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/61a8917ed21e426394bc9e9bbc6f05a0 2024-12-11T20:35:22,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/61a8917ed21e426394bc9e9bbc6f05a0, entries=200, sequenceid=261, filesize=14.3 K 2024-12-11T20:35:22,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/d600d020c8e04b5782ece5d65bdc158c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d600d020c8e04b5782ece5d65bdc158c 2024-12-11T20:35:22,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d600d020c8e04b5782ece5d65bdc158c, entries=150, sequenceid=261, filesize=12.0 K 2024-12-11T20:35:22,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/9841c362fbd145ebb2d85b515e35986e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9841c362fbd145ebb2d85b515e35986e 2024-12-11T20:35:22,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9841c362fbd145ebb2d85b515e35986e, entries=150, sequenceid=261, filesize=12.0 K 2024-12-11T20:35:22,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 320c74933b0c438c8808bab076a15dce in 1259ms, sequenceid=261, compaction requested=true 2024-12-11T20:35:22,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:22,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:22,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:22,815 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:35:22,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:22,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:22,815 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:35:22,817 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48910 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:35:22,817 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/B is initiating minor compaction (all files) 2024-12-11T20:35:22,817 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/B in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,817 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/2776acf147d2441890e963c7bfa97c8a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f9a49dbfe18d476696f478c3f2351d38, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1275246dfd0742fc938de721ae15b415, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d600d020c8e04b5782ece5d65bdc158c] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=47.8 K 2024-12-11T20:35:22,818 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51350 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:35:22,818 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 2776acf147d2441890e963c7bfa97c8a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733949317392 2024-12-11T20:35:22,818 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/A is initiating minor compaction (all files) 2024-12-11T20:35:22,818 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/A in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,818 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/9cbcb387c85d4505994b7ac2451becaa, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/877735eddc754244a3b5263171a4ee9c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1d33f08788514f8189719a224cd493d1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/61a8917ed21e426394bc9e9bbc6f05a0] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=50.1 K 2024-12-11T20:35:22,818 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f9a49dbfe18d476696f478c3f2351d38, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733949318188 2024-12-11T20:35:22,818 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cbcb387c85d4505994b7ac2451becaa, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733949317392 2024-12-11T20:35:22,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:22,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:22,819 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 1275246dfd0742fc938de721ae15b415, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733949319358 2024-12-11T20:35:22,819 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 877735eddc754244a3b5263171a4ee9c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733949318188 2024-12-11T20:35:22,819 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d600d020c8e04b5782ece5d65bdc158c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733949321533 2024-12-11T20:35:22,820 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d33f08788514f8189719a224cd493d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733949319358 2024-12-11T20:35:22,820 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61a8917ed21e426394bc9e9bbc6f05a0, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733949321533 2024-12-11T20:35:22,833 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#B#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:22,833 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/95bf69f31f3b402c807e52feb3355ceb is 50, key is test_row_0/B:col10/1733949321549/Put/seqid=0 2024-12-11T20:35:22,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-11T20:35:22,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,835 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T20:35:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:22,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:22,840 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#A#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:22,841 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/837713e05571499bae0c718843528f3e is 50, key is test_row_0/A:col10/1733949321549/Put/seqid=0 2024-12-11T20:35:22,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/e2e7e3dd80c44c78aeed02ab2c0720f5 is 50, key is test_row_0/A:col10/1733949321607/Put/seqid=0 2024-12-11T20:35:22,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742101_1277 (size=12593) 2024-12-11T20:35:22,869 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/95bf69f31f3b402c807e52feb3355ceb as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/95bf69f31f3b402c807e52feb3355ceb 2024-12-11T20:35:22,874 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 320c74933b0c438c8808bab076a15dce/B of 320c74933b0c438c8808bab076a15dce into 95bf69f31f3b402c807e52feb3355ceb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:22,874 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:22,874 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/B, priority=12, startTime=1733949322815; duration=0sec 2024-12-11T20:35:22,874 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:22,874 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:B 2024-12-11T20:35:22,874 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:35:22,876 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48910 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:35:22,876 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/C is initiating minor compaction (all files) 2024-12-11T20:35:22,876 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/C in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:22,876 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/7984b6d344e547d181a96d4943e6336b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/1101f4f214e74cfc95fab108f209709d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/4974f89c5ec24627b212176eb42e355b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9841c362fbd145ebb2d85b515e35986e] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=47.8 K 2024-12-11T20:35:22,877 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 7984b6d344e547d181a96d4943e6336b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1733949317392 2024-12-11T20:35:22,877 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 1101f4f214e74cfc95fab108f209709d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1733949318188 2024-12-11T20:35:22,877 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 4974f89c5ec24627b212176eb42e355b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1733949319358 2024-12-11T20:35:22,877 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 9841c362fbd145ebb2d85b515e35986e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733949321533 2024-12-11T20:35:22,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742102_1278 (size=12593) 2024-12-11T20:35:22,902 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#C#compaction#233 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:22,902 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/efd5a3e2fd9946929e1dc5efb29510ef is 50, key is test_row_0/C:col10/1733949321549/Put/seqid=0 2024-12-11T20:35:22,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742103_1279 (size=12301) 2024-12-11T20:35:22,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742104_1280 (size=12593) 2024-12-11T20:35:22,930 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/efd5a3e2fd9946929e1dc5efb29510ef as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/efd5a3e2fd9946929e1dc5efb29510ef 2024-12-11T20:35:22,936 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 320c74933b0c438c8808bab076a15dce/C of 320c74933b0c438c8808bab076a15dce into efd5a3e2fd9946929e1dc5efb29510ef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:22,936 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:22,936 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/C, priority=12, startTime=1733949322815; duration=0sec 2024-12-11T20:35:22,936 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:22,936 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:C 2024-12-11T20:35:23,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T20:35:23,295 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/837713e05571499bae0c718843528f3e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/837713e05571499bae0c718843528f3e 2024-12-11T20:35:23,301 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 320c74933b0c438c8808bab076a15dce/A of 320c74933b0c438c8808bab076a15dce into 837713e05571499bae0c718843528f3e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:23,301 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:23,301 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/A, priority=12, startTime=1733949322815; duration=0sec 2024-12-11T20:35:23,301 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:23,301 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:A 2024-12-11T20:35:23,314 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/e2e7e3dd80c44c78aeed02ab2c0720f5 2024-12-11T20:35:23,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/9bb2985c495f4eb19fa6b563c6ca3ea3 is 50, key is test_row_0/B:col10/1733949321607/Put/seqid=0 2024-12-11T20:35:23,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742105_1281 (size=12301) 2024-12-11T20:35:23,356 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/9bb2985c495f4eb19fa6b563c6ca3ea3 2024-12-11T20:35:23,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/9dd3b152b1c34cdc8f38cf61c047cce1 is 50, key is test_row_0/C:col10/1733949321607/Put/seqid=0 2024-12-11T20:35:23,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742106_1282 (size=12301) 2024-12-11T20:35:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:23,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:23,777 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/9dd3b152b1c34cdc8f38cf61c047cce1 2024-12-11T20:35:23,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/e2e7e3dd80c44c78aeed02ab2c0720f5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e2e7e3dd80c44c78aeed02ab2c0720f5 2024-12-11T20:35:23,784 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e2e7e3dd80c44c78aeed02ab2c0720f5, entries=150, sequenceid=279, filesize=12.0 K 2024-12-11T20:35:23,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/9bb2985c495f4eb19fa6b563c6ca3ea3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/9bb2985c495f4eb19fa6b563c6ca3ea3 2024-12-11T20:35:23,789 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/9bb2985c495f4eb19fa6b563c6ca3ea3, entries=150, sequenceid=279, filesize=12.0 K 2024-12-11T20:35:23,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/9dd3b152b1c34cdc8f38cf61c047cce1 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9dd3b152b1c34cdc8f38cf61c047cce1 2024-12-11T20:35:23,793 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9dd3b152b1c34cdc8f38cf61c047cce1, entries=150, sequenceid=279, filesize=12.0 K 2024-12-11T20:35:23,793 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=60.38 KB/61830 for 320c74933b0c438c8808bab076a15dce in 958ms, sequenceid=279, compaction requested=false 2024-12-11T20:35:23,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:23,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:23,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-11T20:35:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-11T20:35:23,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-11T20:35:23,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7250 sec 2024-12-11T20:35:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:23,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T20:35:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:23,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:23,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7300 sec 2024-12-11T20:35:23,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/adbd0b0616bf45f7ae097a05c116cb4c is 50, key is test_row_0/A:col10/1733949323792/Put/seqid=0 2024-12-11T20:35:23,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742107_1283 (size=12301) 2024-12-11T20:35:23,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/adbd0b0616bf45f7ae097a05c116cb4c 2024-12-11T20:35:23,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/78e7c9c5672747c097cd5fc9e487e999 is 50, key is test_row_0/B:col10/1733949323792/Put/seqid=0 2024-12-11T20:35:23,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742108_1284 (size=12301) 2024-12-11T20:35:23,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:23,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949383853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:23,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:23,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949383967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:24,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:24,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949384177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:24,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/78e7c9c5672747c097cd5fc9e487e999 2024-12-11T20:35:24,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-11T20:35:24,217 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:24,218 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:24,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-11T20:35:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T20:35:24,219 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:24,220 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:24,220 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:24,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/85d7fd1b476d442c850e006d19ae95b1 is 50, key is test_row_0/C:col10/1733949323792/Put/seqid=0 2024-12-11T20:35:24,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742109_1285 (size=12301) 2024-12-11T20:35:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T20:35:24,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-11T20:35:24,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:24,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:24,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:24,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:24,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:24,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949384487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:24,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:24,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-11T20:35:24,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:24,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:24,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:24,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:24,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:24,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T20:35:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49266 deadline: 1733949384577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:24,579 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:24,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49298 deadline: 1733949384596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:24,598 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_0' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:24,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49322 deadline: 1733949384607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:24,609 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:24,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/85d7fd1b476d442c850e006d19ae95b1 2024-12-11T20:35:24,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949384627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:24,630 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:24,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:24,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/adbd0b0616bf45f7ae097a05c116cb4c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/adbd0b0616bf45f7ae097a05c116cb4c 2024-12-11T20:35:24,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/adbd0b0616bf45f7ae097a05c116cb4c, entries=150, sequenceid=295, filesize=12.0 K 2024-12-11T20:35:24,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/78e7c9c5672747c097cd5fc9e487e999 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/78e7c9c5672747c097cd5fc9e487e999 2024-12-11T20:35:24,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/78e7c9c5672747c097cd5fc9e487e999, entries=150, sequenceid=295, filesize=12.0 K 2024-12-11T20:35:24,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/85d7fd1b476d442c850e006d19ae95b1 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/85d7fd1b476d442c850e006d19ae95b1 2024-12-11T20:35:24,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/85d7fd1b476d442c850e006d19ae95b1, entries=150, sequenceid=295, filesize=12.0 K 2024-12-11T20:35:24,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 320c74933b0c438c8808bab076a15dce in 852ms, sequenceid=295, compaction requested=true 2024-12-11T20:35:24,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:24,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:24,649 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:24,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:24,649 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:24,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:24,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:24,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:24,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:24,653 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:24,653 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/A is initiating minor compaction (all files) 2024-12-11T20:35:24,653 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/A in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:24,653 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/837713e05571499bae0c718843528f3e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e2e7e3dd80c44c78aeed02ab2c0720f5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/adbd0b0616bf45f7ae097a05c116cb4c] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.3 K 2024-12-11T20:35:24,654 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 837713e05571499bae0c718843528f3e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733949321533 2024-12-11T20:35:24,654 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2e7e3dd80c44c78aeed02ab2c0720f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733949321607 2024-12-11T20:35:24,654 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting adbd0b0616bf45f7ae097a05c116cb4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733949323788 2024-12-11T20:35:24,655 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:24,655 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/B is initiating minor compaction (all files) 2024-12-11T20:35:24,655 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/B in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:24,655 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/95bf69f31f3b402c807e52feb3355ceb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/9bb2985c495f4eb19fa6b563c6ca3ea3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/78e7c9c5672747c097cd5fc9e487e999] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.3 K 2024-12-11T20:35:24,656 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 95bf69f31f3b402c807e52feb3355ceb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733949321533 2024-12-11T20:35:24,656 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bb2985c495f4eb19fa6b563c6ca3ea3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733949321607 2024-12-11T20:35:24,656 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 78e7c9c5672747c097cd5fc9e487e999, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733949323788 2024-12-11T20:35:24,664 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#A#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:24,665 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/102bf94de6e842af8fa89378f7de6179 is 50, key is test_row_0/A:col10/1733949323792/Put/seqid=0 2024-12-11T20:35:24,677 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:24,677 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/a868c534a86e49238de9f6ef73e826c5 is 50, key is test_row_0/B:col10/1733949323792/Put/seqid=0 2024-12-11T20:35:24,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-11T20:35:24,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:24,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742110_1286 (size=12745) 2024-12-11T20:35:24,679 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-11T20:35:24,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:24,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:24,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:24,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:24,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:24,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:24,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/afb481a6a3fa44bb83fee0da3a4971ee is 50, key is test_row_0/A:col10/1733949323828/Put/seqid=0 2024-12-11T20:35:24,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742111_1287 (size=12745) 2024-12-11T20:35:24,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742112_1288 (size=12301) 2024-12-11T20:35:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T20:35:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:25,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:25,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949385033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:25,086 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/102bf94de6e842af8fa89378f7de6179 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/102bf94de6e842af8fa89378f7de6179 2024-12-11T20:35:25,095 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/A of 320c74933b0c438c8808bab076a15dce into 102bf94de6e842af8fa89378f7de6179(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:25,095 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:25,096 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/A, priority=13, startTime=1733949324649; duration=0sec 2024-12-11T20:35:25,096 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:25,096 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:A 2024-12-11T20:35:25,096 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:25,098 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:25,098 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/C is initiating minor compaction (all files) 2024-12-11T20:35:25,098 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/C in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:25,098 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/efd5a3e2fd9946929e1dc5efb29510ef, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9dd3b152b1c34cdc8f38cf61c047cce1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/85d7fd1b476d442c850e006d19ae95b1] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.3 K 2024-12-11T20:35:25,099 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd5a3e2fd9946929e1dc5efb29510ef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1733949321533 2024-12-11T20:35:25,099 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dd3b152b1c34cdc8f38cf61c047cce1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1733949321607 2024-12-11T20:35:25,099 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85d7fd1b476d442c850e006d19ae95b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733949323788 2024-12-11T20:35:25,103 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/afb481a6a3fa44bb83fee0da3a4971ee 2024-12-11T20:35:25,103 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/a868c534a86e49238de9f6ef73e826c5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a868c534a86e49238de9f6ef73e826c5 2024-12-11T20:35:25,111 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/B of 320c74933b0c438c8808bab076a15dce into a868c534a86e49238de9f6ef73e826c5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:25,111 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:25,111 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/B, priority=13, startTime=1733949324649; duration=0sec 2024-12-11T20:35:25,111 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:25,111 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:B 2024-12-11T20:35:25,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/6ebe906e9cbf476085757b70d27afa78 is 50, key is test_row_0/B:col10/1733949323828/Put/seqid=0 2024-12-11T20:35:25,141 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#C#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:25,142 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/61b448791a644c7781de4cb091db8079 is 50, key is test_row_0/C:col10/1733949323792/Put/seqid=0 2024-12-11T20:35:25,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742113_1289 (size=12301) 2024-12-11T20:35:25,146 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/6ebe906e9cbf476085757b70d27afa78 2024-12-11T20:35:25,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949385147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:25,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:25,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742114_1290 (size=12745) 2024-12-11T20:35:25,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/b6c9f927d3534fb9925d9ddf4cc53b85 is 50, key is test_row_0/C:col10/1733949323828/Put/seqid=0 2024-12-11T20:35:25,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742115_1291 (size=12301) 2024-12-11T20:35:25,166 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/b6c9f927d3534fb9925d9ddf4cc53b85 2024-12-11T20:35:25,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/afb481a6a3fa44bb83fee0da3a4971ee as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/afb481a6a3fa44bb83fee0da3a4971ee 2024-12-11T20:35:25,176 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/afb481a6a3fa44bb83fee0da3a4971ee, entries=150, sequenceid=314, filesize=12.0 K 2024-12-11T20:35:25,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/6ebe906e9cbf476085757b70d27afa78 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ebe906e9cbf476085757b70d27afa78 2024-12-11T20:35:25,182 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ebe906e9cbf476085757b70d27afa78, entries=150, sequenceid=314, filesize=12.0 K 2024-12-11T20:35:25,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/b6c9f927d3534fb9925d9ddf4cc53b85 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/b6c9f927d3534fb9925d9ddf4cc53b85 2024-12-11T20:35:25,187 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/b6c9f927d3534fb9925d9ddf4cc53b85, entries=150, sequenceid=314, filesize=12.0 K 2024-12-11T20:35:25,188 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 320c74933b0c438c8808bab076a15dce in 510ms, sequenceid=314, compaction requested=false 2024-12-11T20:35:25,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:25,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:25,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-11T20:35:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-11T20:35:25,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-11T20:35:25,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 969 msec 2024-12-11T20:35:25,194 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 974 msec 2024-12-11T20:35:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-11T20:35:25,357 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:25,357 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:25,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-11T20:35:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T20:35:25,359 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:25,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:25,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-11T20:35:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:25,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:25,360 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:25,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:25,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/ebd74b6defe845fea27e7b777852ce6e is 50, key is test_row_0/A:col10/1733949325358/Put/seqid=0 2024-12-11T20:35:25,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742116_1292 (size=12301) 2024-12-11T20:35:25,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:25,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949385413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:25,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T20:35:25,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T20:35:25,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:25,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949385527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:25,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:25,560 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/61b448791a644c7781de4cb091db8079 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/61b448791a644c7781de4cb091db8079 2024-12-11T20:35:25,564 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/C of 320c74933b0c438c8808bab076a15dce into 61b448791a644c7781de4cb091db8079(size=12.4 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:25,564 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:25,564 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/C, priority=13, startTime=1733949324650; duration=0sec 2024-12-11T20:35:25,564 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:25,564 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:C 2024-12-11T20:35:25,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T20:35:25,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:25,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:25,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:25,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T20:35:25,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949385737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:25,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:25,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/ebd74b6defe845fea27e7b777852ce6e 2024-12-11T20:35:25,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/551ae87e8c6c4ad09d73fb34512c546d is 50, key is test_row_0/B:col10/1733949325358/Put/seqid=0 2024-12-11T20:35:25,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742117_1293 (size=12301) 2024-12-11T20:35:25,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/551ae87e8c6c4ad09d73fb34512c546d 2024-12-11T20:35:25,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/2a343ab4d34f4af1b886189a9edba715 is 50, key is test_row_0/C:col10/1733949325358/Put/seqid=0 2024-12-11T20:35:25,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742118_1294 (size=12301) 2024-12-11T20:35:25,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T20:35:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:25,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T20:35:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:25,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T20:35:26,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949386047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:26,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T20:35:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:26,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:26,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:26,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/2a343ab4d34f4af1b886189a9edba715 2024-12-11T20:35:26,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/ebd74b6defe845fea27e7b777852ce6e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/ebd74b6defe845fea27e7b777852ce6e 2024-12-11T20:35:26,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/ebd74b6defe845fea27e7b777852ce6e, entries=150, sequenceid=330, filesize=12.0 K 2024-12-11T20:35:26,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/551ae87e8c6c4ad09d73fb34512c546d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/551ae87e8c6c4ad09d73fb34512c546d 2024-12-11T20:35:26,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/551ae87e8c6c4ad09d73fb34512c546d, entries=150, sequenceid=330, filesize=12.0 K 2024-12-11T20:35:26,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/2a343ab4d34f4af1b886189a9edba715 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2a343ab4d34f4af1b886189a9edba715 2024-12-11T20:35:26,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2a343ab4d34f4af1b886189a9edba715, entries=150, sequenceid=330, filesize=12.0 K 2024-12-11T20:35:26,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 320c74933b0c438c8808bab076a15dce in 846ms, sequenceid=330, compaction requested=true 2024-12-11T20:35:26,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:26,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:26,206 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:26,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:26,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:26,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:26,206 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:26,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:26,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:26,207 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:26,207 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:26,207 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/A is initiating minor compaction (all files) 2024-12-11T20:35:26,207 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/B is initiating minor compaction (all files) 2024-12-11T20:35:26,207 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/A in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:26,207 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/B in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:26,207 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a868c534a86e49238de9f6ef73e826c5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ebe906e9cbf476085757b70d27afa78, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/551ae87e8c6c4ad09d73fb34512c546d] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.5 K 2024-12-11T20:35:26,207 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/102bf94de6e842af8fa89378f7de6179, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/afb481a6a3fa44bb83fee0da3a4971ee, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/ebd74b6defe845fea27e7b777852ce6e] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.5 K 2024-12-11T20:35:26,208 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 102bf94de6e842af8fa89378f7de6179, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733949323788 2024-12-11T20:35:26,208 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting a868c534a86e49238de9f6ef73e826c5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733949323788 2024-12-11T20:35:26,208 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ebe906e9cbf476085757b70d27afa78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733949323828 2024-12-11T20:35:26,208 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting afb481a6a3fa44bb83fee0da3a4971ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733949323828 2024-12-11T20:35:26,208 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebd74b6defe845fea27e7b777852ce6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733949325003 2024-12-11T20:35:26,208 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 551ae87e8c6c4ad09d73fb34512c546d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733949325003 2024-12-11T20:35:26,214 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#A#compaction#248 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:26,215 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/a3ced0687628421bb85e7252455edae3 is 50, key is test_row_0/A:col10/1733949325358/Put/seqid=0 2024-12-11T20:35:26,216 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:26,217 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/3e9486baf90446ebb8287acbffe7873a is 50, key is test_row_0/B:col10/1733949325358/Put/seqid=0 2024-12-11T20:35:26,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742119_1295 (size=12847) 2024-12-11T20:35:26,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742120_1296 (size=12847) 2024-12-11T20:35:26,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-11T20:35:26,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:26,278 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T20:35:26,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:26,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:26,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:26,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:26,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:26,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:26,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/b987ba100243466387313b38df89b258 is 50, key is test_row_0/A:col10/1733949325377/Put/seqid=0 2024-12-11T20:35:26,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742121_1297 (size=12301) 2024-12-11T20:35:26,286 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/b987ba100243466387313b38df89b258 2024-12-11T20:35:26,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/834f64e94c654cbaa1d5f8e10de9725b is 50, key is test_row_0/B:col10/1733949325377/Put/seqid=0 2024-12-11T20:35:26,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742122_1298 (size=12301) 2024-12-11T20:35:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T20:35:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:26,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:26,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949386599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:26,628 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/a3ced0687628421bb85e7252455edae3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/a3ced0687628421bb85e7252455edae3 2024-12-11T20:35:26,633 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/3e9486baf90446ebb8287acbffe7873a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/3e9486baf90446ebb8287acbffe7873a 2024-12-11T20:35:26,633 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/A of 320c74933b0c438c8808bab076a15dce into a3ced0687628421bb85e7252455edae3(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:26,633 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:26,633 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/A, priority=13, startTime=1733949326206; duration=0sec 2024-12-11T20:35:26,634 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:26,634 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:A 2024-12-11T20:35:26,634 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:26,635 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:26,635 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/C is initiating minor compaction (all files) 2024-12-11T20:35:26,635 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/C in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:26,635 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/61b448791a644c7781de4cb091db8079, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/b6c9f927d3534fb9925d9ddf4cc53b85, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2a343ab4d34f4af1b886189a9edba715] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.5 K 2024-12-11T20:35:26,635 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61b448791a644c7781de4cb091db8079, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1733949323788 2024-12-11T20:35:26,636 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6c9f927d3534fb9925d9ddf4cc53b85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733949323828 2024-12-11T20:35:26,636 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a343ab4d34f4af1b886189a9edba715, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733949325003 2024-12-11T20:35:26,639 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/B of 320c74933b0c438c8808bab076a15dce into 3e9486baf90446ebb8287acbffe7873a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:26,639 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:26,639 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/B, priority=13, startTime=1733949326206; duration=0sec 2024-12-11T20:35:26,639 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:26,639 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:B 2024-12-11T20:35:26,657 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#C#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:26,657 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/e9a0f6ce07884c70a27396dca6a1728e is 50, key is test_row_0/C:col10/1733949325358/Put/seqid=0 2024-12-11T20:35:26,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742123_1299 (size=12847) 2024-12-11T20:35:26,695 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/834f64e94c654cbaa1d5f8e10de9725b 2024-12-11T20:35:26,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/fc353fac98ed4aaf943ac523471044e5 is 50, key is test_row_0/C:col10/1733949325377/Put/seqid=0 2024-12-11T20:35:26,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742124_1300 (size=12301) 2024-12-11T20:35:26,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949386707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:26,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:26,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949386917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:26,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:26,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:26,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:27,081 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/e9a0f6ce07884c70a27396dca6a1728e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/e9a0f6ce07884c70a27396dca6a1728e 2024-12-11T20:35:27,085 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/C of 320c74933b0c438c8808bab076a15dce into e9a0f6ce07884c70a27396dca6a1728e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:27,085 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:27,085 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/C, priority=13, startTime=1733949326206; duration=0sec 2024-12-11T20:35:27,085 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:27,085 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:C 2024-12-11T20:35:27,106 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/fc353fac98ed4aaf943ac523471044e5 2024-12-11T20:35:27,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/b987ba100243466387313b38df89b258 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/b987ba100243466387313b38df89b258 2024-12-11T20:35:27,115 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/b987ba100243466387313b38df89b258, entries=150, sequenceid=349, filesize=12.0 K 2024-12-11T20:35:27,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/834f64e94c654cbaa1d5f8e10de9725b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/834f64e94c654cbaa1d5f8e10de9725b 2024-12-11T20:35:27,121 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/834f64e94c654cbaa1d5f8e10de9725b, entries=150, sequenceid=349, filesize=12.0 K 2024-12-11T20:35:27,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/fc353fac98ed4aaf943ac523471044e5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/fc353fac98ed4aaf943ac523471044e5 2024-12-11T20:35:27,126 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/fc353fac98ed4aaf943ac523471044e5, entries=150, sequenceid=349, filesize=12.0 K 2024-12-11T20:35:27,126 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 320c74933b0c438c8808bab076a15dce in 849ms, sequenceid=349, compaction requested=false 2024-12-11T20:35:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-11T20:35:27,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-11T20:35:27,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-11T20:35:27,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7800 sec 2024-12-11T20:35:27,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.7860 sec 2024-12-11T20:35:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:27,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T20:35:27,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:27,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:27,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:27,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:27,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:27,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:27,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/bb1ffd15e65347bc90e7f8fa31d94033 is 50, key is test_row_0/A:col10/1733949326597/Put/seqid=0 2024-12-11T20:35:27,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742125_1301 (size=14741) 2024-12-11T20:35:27,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/bb1ffd15e65347bc90e7f8fa31d94033 2024-12-11T20:35:27,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/1e2448018252454ab41690aef9b70508 is 50, key is test_row_0/B:col10/1733949326597/Put/seqid=0 2024-12-11T20:35:27,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742126_1302 (size=12301) 2024-12-11T20:35:27,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949387286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:27,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:27,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949387397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-11T20:35:27,511 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:27,511 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-11T20:35:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T20:35:27,513 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:27,513 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:27,514 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:27,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949387606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:27,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T20:35:27,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-11T20:35:27,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:27,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:27,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:27,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:27,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:27,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/1e2448018252454ab41690aef9b70508 2024-12-11T20:35:27,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/c0edc306d5f74ea7bfe5e7d812815d4d is 50, key is test_row_0/C:col10/1733949326597/Put/seqid=0 2024-12-11T20:35:27,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742127_1303 (size=12301) 2024-12-11T20:35:27,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-11T20:35:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:27,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T20:35:27,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949387917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:27,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:27,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-11T20:35:27,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:27,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:27,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:28,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/c0edc306d5f74ea7bfe5e7d812815d4d 2024-12-11T20:35:28,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-11T20:35:28,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:28,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:28,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:28,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:28,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:28,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/bb1ffd15e65347bc90e7f8fa31d94033 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bb1ffd15e65347bc90e7f8fa31d94033 2024-12-11T20:35:28,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bb1ffd15e65347bc90e7f8fa31d94033, entries=200, sequenceid=367, filesize=14.4 K 2024-12-11T20:35:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/1e2448018252454ab41690aef9b70508 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1e2448018252454ab41690aef9b70508 2024-12-11T20:35:28,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1e2448018252454ab41690aef9b70508, entries=150, sequenceid=367, filesize=12.0 K 2024-12-11T20:35:28,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/c0edc306d5f74ea7bfe5e7d812815d4d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/c0edc306d5f74ea7bfe5e7d812815d4d 2024-12-11T20:35:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T20:35:28,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/c0edc306d5f74ea7bfe5e7d812815d4d, entries=150, sequenceid=367, filesize=12.0 K 2024-12-11T20:35:28,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 320c74933b0c438c8808bab076a15dce in 911ms, sequenceid=367, compaction requested=true 2024-12-11T20:35:28,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:28,140 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:28,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:28,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:28,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:28,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:28,141 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:28,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:28,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:28,141 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:28,141 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/A is initiating minor compaction (all files) 2024-12-11T20:35:28,141 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/A in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:28,142 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/a3ced0687628421bb85e7252455edae3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/b987ba100243466387313b38df89b258, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bb1ffd15e65347bc90e7f8fa31d94033] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=39.0 K 2024-12-11T20:35:28,142 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:28,142 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/B is initiating minor compaction (all files) 2024-12-11T20:35:28,142 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3ced0687628421bb85e7252455edae3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733949325003 2024-12-11T20:35:28,142 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/B in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:28,142 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/3e9486baf90446ebb8287acbffe7873a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/834f64e94c654cbaa1d5f8e10de9725b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1e2448018252454ab41690aef9b70508] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.6 K 2024-12-11T20:35:28,142 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b987ba100243466387313b38df89b258, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733949325377 2024-12-11T20:35:28,142 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb1ffd15e65347bc90e7f8fa31d94033, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733949326590 2024-12-11T20:35:28,142 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e9486baf90446ebb8287acbffe7873a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733949325003 2024-12-11T20:35:28,143 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 834f64e94c654cbaa1d5f8e10de9725b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733949325377 2024-12-11T20:35:28,143 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e2448018252454ab41690aef9b70508, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733949326590 2024-12-11T20:35:28,161 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#B#compaction#257 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:28,161 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/ec3c7d50e4af47e8bdf90a5be311fb78 is 50, key is test_row_0/B:col10/1733949326597/Put/seqid=0 2024-12-11T20:35:28,168 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:28,169 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/fea069d035bb4577990005dba3f30430 is 50, key is test_row_0/A:col10/1733949326597/Put/seqid=0 2024-12-11T20:35:28,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742128_1304 (size=12949) 2024-12-11T20:35:28,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742129_1305 (size=12949) 2024-12-11T20:35:28,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-11T20:35:28,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:28,277 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T20:35:28,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:28,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:28,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:28,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:28,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:28,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:28,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/bc173ffd99604570a96b6b74046cb999 is 50, key is test_row_0/A:col10/1733949327284/Put/seqid=0 2024-12-11T20:35:28,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742130_1306 (size=12301) 2024-12-11T20:35:28,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:28,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:28,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:28,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949388474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:28,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:28,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:28,575 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/ec3c7d50e4af47e8bdf90a5be311fb78 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/ec3c7d50e4af47e8bdf90a5be311fb78 2024-12-11T20:35:28,579 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/B of 320c74933b0c438c8808bab076a15dce into ec3c7d50e4af47e8bdf90a5be311fb78(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:28,580 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:28,580 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/B, priority=13, startTime=1733949328141; duration=0sec 2024-12-11T20:35:28,580 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:28,580 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:B 2024-12-11T20:35:28,580 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:28,581 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:28,581 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/C is initiating minor compaction (all files) 2024-12-11T20:35:28,581 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/C in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:28,581 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/e9a0f6ce07884c70a27396dca6a1728e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/fc353fac98ed4aaf943ac523471044e5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/c0edc306d5f74ea7bfe5e7d812815d4d] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.6 K 2024-12-11T20:35:28,581 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting e9a0f6ce07884c70a27396dca6a1728e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1733949325003 2024-12-11T20:35:28,581 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting fc353fac98ed4aaf943ac523471044e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733949325377 2024-12-11T20:35:28,581 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/fea069d035bb4577990005dba3f30430 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/fea069d035bb4577990005dba3f30430 2024-12-11T20:35:28,582 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c0edc306d5f74ea7bfe5e7d812815d4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733949326590 2024-12-11T20:35:28,586 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/A of 320c74933b0c438c8808bab076a15dce into fea069d035bb4577990005dba3f30430(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:28,586 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:28,586 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/A, priority=13, startTime=1733949328140; duration=0sec 2024-12-11T20:35:28,586 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:28,586 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:A 2024-12-11T20:35:28,589 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:28,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949388587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:28,590 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/0bc729a206ee484ab6a77fed489ef8f5 is 50, key is test_row_0/C:col10/1733949326597/Put/seqid=0 2024-12-11T20:35:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:28,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:28,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742131_1307 (size=12949) 2024-12-11T20:35:28,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T20:35:28,685 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/bc173ffd99604570a96b6b74046cb999 2024-12-11T20:35:28,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/556d7254ab7a4553a9e1985bdfc57086 is 50, key is test_row_0/B:col10/1733949327284/Put/seqid=0 2024-12-11T20:35:28,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742132_1308 (size=12301) 2024-12-11T20:35:28,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949388796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:28,998 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/0bc729a206ee484ab6a77fed489ef8f5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/0bc729a206ee484ab6a77fed489ef8f5 2024-12-11T20:35:29,002 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/C of 320c74933b0c438c8808bab076a15dce into 0bc729a206ee484ab6a77fed489ef8f5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:29,002 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:29,002 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/C, priority=13, startTime=1733949328141; duration=0sec 2024-12-11T20:35:29,002 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:29,002 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:C 2024-12-11T20:35:29,097 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/556d7254ab7a4553a9e1985bdfc57086 2024-12-11T20:35:29,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949389107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:29,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:29,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/6f719f77aea04a44b50d1b32d9bff1e5 is 50, key is test_row_0/C:col10/1733949327284/Put/seqid=0 2024-12-11T20:35:29,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742133_1309 (size=12301) 2024-12-11T20:35:29,122 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/6f719f77aea04a44b50d1b32d9bff1e5 2024-12-11T20:35:29,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/bc173ffd99604570a96b6b74046cb999 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bc173ffd99604570a96b6b74046cb999 2024-12-11T20:35:29,132 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bc173ffd99604570a96b6b74046cb999, entries=150, sequenceid=384, filesize=12.0 K 2024-12-11T20:35:29,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/556d7254ab7a4553a9e1985bdfc57086 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/556d7254ab7a4553a9e1985bdfc57086 2024-12-11T20:35:29,139 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/556d7254ab7a4553a9e1985bdfc57086, entries=150, sequenceid=384, filesize=12.0 K 2024-12-11T20:35:29,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/6f719f77aea04a44b50d1b32d9bff1e5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6f719f77aea04a44b50d1b32d9bff1e5 2024-12-11T20:35:29,146 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6f719f77aea04a44b50d1b32d9bff1e5, entries=150, sequenceid=384, filesize=12.0 K 2024-12-11T20:35:29,147 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 320c74933b0c438c8808bab076a15dce in 870ms, sequenceid=384, compaction requested=false 2024-12-11T20:35:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:29,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-11T20:35:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-11T20:35:29,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-11T20:35:29,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6350 sec 2024-12-11T20:35:29,156 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.6430 sec 2024-12-11T20:35:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:29,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:35:29,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:29,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:29,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:29,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:29,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:29,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:29,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/07eb82cb30a649f28f0ef8a6232bd253 is 50, key is test_row_0/A:col10/1733949328464/Put/seqid=0 2024-12-11T20:35:29,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742134_1310 (size=12301) 2024-12-11T20:35:29,652 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/07eb82cb30a649f28f0ef8a6232bd253 2024-12-11T20:35:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-11T20:35:29,657 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:29,657 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-11T20:35:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-11T20:35:29,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/a189badeffac4d9287192f3c3ccadb6a is 50, key is test_row_0/B:col10/1733949328464/Put/seqid=0 2024-12-11T20:35:29,664 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:29,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742135_1311 (size=12301) 2024-12-11T20:35:29,665 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:29,665 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:29,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/a189badeffac4d9287192f3c3ccadb6a 2024-12-11T20:35:29,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/6e9a66610e684efd91c1025dfb5c78e5 is 50, key is test_row_0/C:col10/1733949328464/Put/seqid=0 2024-12-11T20:35:29,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742136_1312 (size=12301) 2024-12-11T20:35:29,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949389700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:29,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:29,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-11T20:35:29,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949389807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:29,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:29,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:29,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:29,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-11T20:35:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:29,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:29,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:29,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:29,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-11T20:35:29,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:29,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:29,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:29,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:29,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:29,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-11T20:35:30,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:30,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949390017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:30,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:30,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/6e9a66610e684efd91c1025dfb5c78e5 2024-12-11T20:35:30,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/07eb82cb30a649f28f0ef8a6232bd253 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/07eb82cb30a649f28f0ef8a6232bd253 2024-12-11T20:35:30,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/07eb82cb30a649f28f0ef8a6232bd253, entries=150, sequenceid=403, filesize=12.0 K 2024-12-11T20:35:30,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/a189badeffac4d9287192f3c3ccadb6a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a189badeffac4d9287192f3c3ccadb6a 2024-12-11T20:35:30,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a189badeffac4d9287192f3c3ccadb6a, entries=150, sequenceid=403, filesize=12.0 K 2024-12-11T20:35:30,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/6e9a66610e684efd91c1025dfb5c78e5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6e9a66610e684efd91c1025dfb5c78e5 2024-12-11T20:35:30,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6e9a66610e684efd91c1025dfb5c78e5, entries=150, sequenceid=403, filesize=12.0 K 2024-12-11T20:35:30,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 320c74933b0c438c8808bab076a15dce in 476ms, sequenceid=403, compaction requested=true 2024-12-11T20:35:30,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:30,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:30,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:30,104 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:30,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:30,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:30,104 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:30,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:30,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:30,105 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:30,105 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:30,105 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/B is initiating minor compaction (all files) 2024-12-11T20:35:30,105 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/A is initiating minor compaction (all files) 2024-12-11T20:35:30,105 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/B in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,105 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/A in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,105 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/fea069d035bb4577990005dba3f30430, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bc173ffd99604570a96b6b74046cb999, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/07eb82cb30a649f28f0ef8a6232bd253] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.7 K 2024-12-11T20:35:30,105 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/ec3c7d50e4af47e8bdf90a5be311fb78, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/556d7254ab7a4553a9e1985bdfc57086, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a189badeffac4d9287192f3c3ccadb6a] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.7 K 2024-12-11T20:35:30,105 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting fea069d035bb4577990005dba3f30430, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733949326590 2024-12-11T20:35:30,105 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting ec3c7d50e4af47e8bdf90a5be311fb78, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733949326590 2024-12-11T20:35:30,106 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 556d7254ab7a4553a9e1985bdfc57086, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733949327262 2024-12-11T20:35:30,106 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc173ffd99604570a96b6b74046cb999, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733949327262 2024-12-11T20:35:30,106 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting a189badeffac4d9287192f3c3ccadb6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733949328459 2024-12-11T20:35:30,106 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07eb82cb30a649f28f0ef8a6232bd253, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733949328459 2024-12-11T20:35:30,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-11T20:35:30,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,123 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:35:30,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:30,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:30,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:30,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:30,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:30,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:30,132 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#B#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:30,133 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/f2effdb2f78846c786ba0da74fada3ba is 50, key is test_row_0/B:col10/1733949328464/Put/seqid=0 2024-12-11T20:35:30,135 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:30,136 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/49350f5ec7b4464b8489caf17eae421f is 50, key is test_row_0/A:col10/1733949328464/Put/seqid=0 2024-12-11T20:35:30,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/434af51b7b8d423ea69719f46f2673d0 is 50, key is test_row_0/A:col10/1733949329674/Put/seqid=0 2024-12-11T20:35:30,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742138_1314 (size=13051) 2024-12-11T20:35:30,162 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/49350f5ec7b4464b8489caf17eae421f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/49350f5ec7b4464b8489caf17eae421f 2024-12-11T20:35:30,172 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/A of 320c74933b0c438c8808bab076a15dce into 49350f5ec7b4464b8489caf17eae421f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:30,172 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:30,172 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/A, priority=13, startTime=1733949330104; duration=0sec 2024-12-11T20:35:30,172 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:30,172 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:A 2024-12-11T20:35:30,172 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:30,174 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:30,174 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/C is initiating minor compaction (all files) 2024-12-11T20:35:30,174 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/C in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,174 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/0bc729a206ee484ab6a77fed489ef8f5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6f719f77aea04a44b50d1b32d9bff1e5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6e9a66610e684efd91c1025dfb5c78e5] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.7 K 2024-12-11T20:35:30,175 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc729a206ee484ab6a77fed489ef8f5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1733949326590 2024-12-11T20:35:30,175 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f719f77aea04a44b50d1b32d9bff1e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1733949327262 2024-12-11T20:35:30,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742137_1313 (size=13051) 2024-12-11T20:35:30,176 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e9a66610e684efd91c1025dfb5c78e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733949328459 2024-12-11T20:35:30,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742139_1315 (size=12301) 2024-12-11T20:35:30,177 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/434af51b7b8d423ea69719f46f2673d0 2024-12-11T20:35:30,181 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/f2effdb2f78846c786ba0da74fada3ba as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f2effdb2f78846c786ba0da74fada3ba 2024-12-11T20:35:30,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/6ce4ac96611d4ac1a1bd5e4bd9148650 is 50, key is test_row_0/B:col10/1733949329674/Put/seqid=0 2024-12-11T20:35:30,189 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:30,190 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/f0700fa86c6048b3931cf8311e45f9b6 is 50, key is test_row_0/C:col10/1733949328464/Put/seqid=0 2024-12-11T20:35:30,191 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/B of 320c74933b0c438c8808bab076a15dce into f2effdb2f78846c786ba0da74fada3ba(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:30,191 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:30,191 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/B, priority=13, startTime=1733949330104; duration=0sec 2024-12-11T20:35:30,191 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:30,191 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:B 2024-12-11T20:35:30,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742140_1316 (size=13051) 2024-12-11T20:35:30,211 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/f0700fa86c6048b3931cf8311e45f9b6 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/f0700fa86c6048b3931cf8311e45f9b6 2024-12-11T20:35:30,216 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/C of 320c74933b0c438c8808bab076a15dce into f0700fa86c6048b3931cf8311e45f9b6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:30,216 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:30,216 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/C, priority=13, startTime=1733949330104; duration=0sec 2024-12-11T20:35:30,216 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:30,216 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:C 2024-12-11T20:35:30,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742141_1317 (size=12301) 2024-12-11T20:35:30,219 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/6ce4ac96611d4ac1a1bd5e4bd9148650 2024-12-11T20:35:30,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/3c53ec4ec1804ba18ca17f5049cd76f6 is 50, key is test_row_0/C:col10/1733949329674/Put/seqid=0 2024-12-11T20:35:30,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742142_1318 (size=12301) 2024-12-11T20:35:30,229 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/3c53ec4ec1804ba18ca17f5049cd76f6 2024-12-11T20:35:30,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/434af51b7b8d423ea69719f46f2673d0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/434af51b7b8d423ea69719f46f2673d0 2024-12-11T20:35:30,236 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/434af51b7b8d423ea69719f46f2673d0, entries=150, sequenceid=419, filesize=12.0 K 2024-12-11T20:35:30,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/6ce4ac96611d4ac1a1bd5e4bd9148650 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ce4ac96611d4ac1a1bd5e4bd9148650 2024-12-11T20:35:30,245 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ce4ac96611d4ac1a1bd5e4bd9148650, entries=150, sequenceid=419, filesize=12.0 K 2024-12-11T20:35:30,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/3c53ec4ec1804ba18ca17f5049cd76f6 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3c53ec4ec1804ba18ca17f5049cd76f6 2024-12-11T20:35:30,253 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3c53ec4ec1804ba18ca17f5049cd76f6, entries=150, sequenceid=419, filesize=12.0 K 2024-12-11T20:35:30,255 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 320c74933b0c438c8808bab076a15dce in 131ms, sequenceid=419, compaction requested=false 2024-12-11T20:35:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-11T20:35:30,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-11T20:35:30,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-11T20:35:30,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 592 msec 2024-12-11T20:35:30,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 603 msec 2024-12-11T20:35:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-11T20:35:30,288 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:30,288 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:30,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-11T20:35:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-11T20:35:30,292 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:30,294 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:30,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:30,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:30,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:35:30,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:30,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:30,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:30,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:30,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:30,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:30,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/84ccd53fdb9b4061b563a568289e2f7e is 50, key is test_row_0/A:col10/1733949330345/Put/seqid=0 2024-12-11T20:35:30,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742143_1319 (size=12301) 2024-12-11T20:35:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-11T20:35:30,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-11T20:35:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949330449, max timestamp=1733949330461Num uniques 859; 2024-12-11T20:35:30,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:30,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949330449, max timestamp=1733949330461Num uniques 859; 2024-12-11T20:35:30,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:30,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949330449, max timestamp=1733949330461Num uniques 859; 2024-12-11T20:35:30,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:30,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949390537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:30,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-11T20:35:30,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:30,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-11T20:35:30,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949390647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:30,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:30,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-11T20:35:30,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:30,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/84ccd53fdb9b4061b563a568289e2f7e 2024-12-11T20:35:30,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/f133cbdd243145e88c4291a189d5d371 is 50, key is test_row_0/B:col10/1733949330345/Put/seqid=0 2024-12-11T20:35:30,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742144_1320 (size=12301) 2024-12-11T20:35:30,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949390857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:30,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-11T20:35:30,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:30,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:30,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:30,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-11T20:35:31,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-11T20:35:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:31,059 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:31,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:31,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:31,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949391167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:31,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:31,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/f133cbdd243145e88c4291a189d5d371 2024-12-11T20:35:31,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/058a5a7a696c49a397c0352f1a1f137a is 50, key is test_row_0/C:col10/1733949330345/Put/seqid=0 2024-12-11T20:35:31,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742145_1321 (size=12301) 2024-12-11T20:35:31,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/058a5a7a696c49a397c0352f1a1f137a 2024-12-11T20:35:31,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-11T20:35:31,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:31,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:31,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:31,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:31,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:31,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:31,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/84ccd53fdb9b4061b563a568289e2f7e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/84ccd53fdb9b4061b563a568289e2f7e 2024-12-11T20:35:31,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/84ccd53fdb9b4061b563a568289e2f7e, entries=150, sequenceid=432, filesize=12.0 K 2024-12-11T20:35:31,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/f133cbdd243145e88c4291a189d5d371 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f133cbdd243145e88c4291a189d5d371 2024-12-11T20:35:31,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f133cbdd243145e88c4291a189d5d371, entries=150, sequenceid=432, filesize=12.0 K 2024-12-11T20:35:31,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/058a5a7a696c49a397c0352f1a1f137a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/058a5a7a696c49a397c0352f1a1f137a 2024-12-11T20:35:31,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/058a5a7a696c49a397c0352f1a1f137a, entries=150, sequenceid=432, filesize=12.0 K 2024-12-11T20:35:31,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 320c74933b0c438c8808bab076a15dce in 884ms, sequenceid=432, compaction requested=true 2024-12-11T20:35:31,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:31,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:31,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:31,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:31,230 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:31,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:31,230 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:31,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:31,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:31,231 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:31,231 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:31,231 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/B is initiating minor compaction (all files) 2024-12-11T20:35:31,231 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/A is initiating minor compaction (all files) 2024-12-11T20:35:31,231 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/B in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:31,231 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/A in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:31,231 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f2effdb2f78846c786ba0da74fada3ba, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ce4ac96611d4ac1a1bd5e4bd9148650, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f133cbdd243145e88c4291a189d5d371] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.8 K 2024-12-11T20:35:31,231 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/49350f5ec7b4464b8489caf17eae421f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/434af51b7b8d423ea69719f46f2673d0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/84ccd53fdb9b4061b563a568289e2f7e] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.8 K 2024-12-11T20:35:31,231 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f2effdb2f78846c786ba0da74fada3ba, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733949328459 2024-12-11T20:35:31,231 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49350f5ec7b4464b8489caf17eae421f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733949328459 2024-12-11T20:35:31,231 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ce4ac96611d4ac1a1bd5e4bd9148650, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733949329674 2024-12-11T20:35:31,232 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 434af51b7b8d423ea69719f46f2673d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733949329674 2024-12-11T20:35:31,232 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f133cbdd243145e88c4291a189d5d371, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733949330334 2024-12-11T20:35:31,232 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84ccd53fdb9b4061b563a568289e2f7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733949330334 2024-12-11T20:35:31,237 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#A#compaction#275 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:31,237 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#B#compaction#276 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:31,238 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/a2f7cbd414834621852d270b36234c7d is 50, key is test_row_0/B:col10/1733949330345/Put/seqid=0 2024-12-11T20:35:31,238 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/2ca7939b69524c9aa3b2d374166c7290 is 50, key is test_row_0/A:col10/1733949330345/Put/seqid=0 2024-12-11T20:35:31,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742146_1322 (size=13153) 2024-12-11T20:35:31,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742147_1323 (size=13153) 2024-12-11T20:35:31,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-11T20:35:31,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:31,366 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-11T20:35:31,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:31,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:31,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:31,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:31,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:31,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:31,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/dcd88ff0e09547878aa8dab7c0afb62c is 50, key is test_row_0/A:col10/1733949330535/Put/seqid=0 2024-12-11T20:35:31,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742148_1324 (size=12301) 2024-12-11T20:35:31,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-11T20:35:31,648 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/a2f7cbd414834621852d270b36234c7d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a2f7cbd414834621852d270b36234c7d 2024-12-11T20:35:31,648 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/2ca7939b69524c9aa3b2d374166c7290 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/2ca7939b69524c9aa3b2d374166c7290 2024-12-11T20:35:31,653 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/A of 320c74933b0c438c8808bab076a15dce into 2ca7939b69524c9aa3b2d374166c7290(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:31,653 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/B of 320c74933b0c438c8808bab076a15dce into a2f7cbd414834621852d270b36234c7d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:31,654 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:31,654 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:31,654 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/B, priority=13, startTime=1733949331230; duration=0sec 2024-12-11T20:35:31,654 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/A, priority=13, startTime=1733949331230; duration=0sec 2024-12-11T20:35:31,654 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:31,654 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:B 2024-12-11T20:35:31,654 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:31,654 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:A 2024-12-11T20:35:31,654 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:31,655 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:31,655 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/C is initiating minor compaction (all files) 2024-12-11T20:35:31,655 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/C in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:31,655 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/f0700fa86c6048b3931cf8311e45f9b6, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3c53ec4ec1804ba18ca17f5049cd76f6, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/058a5a7a696c49a397c0352f1a1f137a] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.8 K 2024-12-11T20:35:31,655 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f0700fa86c6048b3931cf8311e45f9b6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733949328459 2024-12-11T20:35:31,656 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c53ec4ec1804ba18ca17f5049cd76f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1733949329674 2024-12-11T20:35:31,656 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 058a5a7a696c49a397c0352f1a1f137a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733949330334 2024-12-11T20:35:31,662 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#C#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:31,662 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/63cc11b12226468594bcb47aafd01ac2 is 50, key is test_row_0/C:col10/1733949330345/Put/seqid=0 2024-12-11T20:35:31,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742149_1325 (size=13153) 2024-12-11T20:35:31,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:31,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:31,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949391699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:31,787 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/dcd88ff0e09547878aa8dab7c0afb62c 2024-12-11T20:35:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/8766b4d6c8b842a49cf224c5c972f9e7 is 50, key is test_row_0/B:col10/1733949330535/Put/seqid=0 2024-12-11T20:35:31,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742150_1326 (size=12301) 2024-12-11T20:35:31,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:31,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949391806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:31,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:32,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 296 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949392017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:32,072 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/63cc11b12226468594bcb47aafd01ac2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/63cc11b12226468594bcb47aafd01ac2 2024-12-11T20:35:32,076 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/C of 320c74933b0c438c8808bab076a15dce into 63cc11b12226468594bcb47aafd01ac2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:32,076 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:32,077 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/C, priority=13, startTime=1733949331230; duration=0sec 2024-12-11T20:35:32,077 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:32,077 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:C 2024-12-11T20:35:32,197 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/8766b4d6c8b842a49cf224c5c972f9e7 2024-12-11T20:35:32,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/095878b1b0184abfa601fc353d637da2 is 50, key is test_row_0/C:col10/1733949330535/Put/seqid=0 2024-12-11T20:35:32,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742151_1327 (size=12301) 2024-12-11T20:35:32,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949392326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:32,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-11T20:35:32,608 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/095878b1b0184abfa601fc353d637da2 2024-12-11T20:35:32,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/dcd88ff0e09547878aa8dab7c0afb62c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/dcd88ff0e09547878aa8dab7c0afb62c 2024-12-11T20:35:32,615 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/dcd88ff0e09547878aa8dab7c0afb62c, entries=150, sequenceid=469, filesize=12.0 K 2024-12-11T20:35:32,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/8766b4d6c8b842a49cf224c5c972f9e7 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/8766b4d6c8b842a49cf224c5c972f9e7 2024-12-11T20:35:32,619 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/8766b4d6c8b842a49cf224c5c972f9e7, entries=150, sequenceid=469, filesize=12.0 K 2024-12-11T20:35:32,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/095878b1b0184abfa601fc353d637da2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/095878b1b0184abfa601fc353d637da2 2024-12-11T20:35:32,623 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/095878b1b0184abfa601fc353d637da2, entries=150, sequenceid=469, filesize=12.0 K 2024-12-11T20:35:32,624 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 320c74933b0c438c8808bab076a15dce in 1258ms, sequenceid=469, compaction requested=false 2024-12-11T20:35:32,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:32,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:32,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-11T20:35:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-11T20:35:32,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-11T20:35:32,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3310 sec 2024-12-11T20:35:32,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.3390 sec 2024-12-11T20:35:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:32,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T20:35:32,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:32,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:32,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:32,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:32,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:32,839 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:32,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/707211a678324ccfa39c10623c63f6aa is 50, key is test_row_0/A:col10/1733949331697/Put/seqid=0 2024-12-11T20:35:32,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742152_1328 (size=14741) 2024-12-11T20:35:32,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949332899, max timestamp=1733949332911Num uniques 859; 2024-12-11T20:35:32,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:32,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949332899, max timestamp=1733949332911Num uniques 859; 2024-12-11T20:35:32,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949332899, max timestamp=1733949332911Num uniques 859; 2024-12-11T20:35:32,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:32,914 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:35:32,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:32,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949392983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:32,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:33,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:33,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949393096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:33,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:33,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/707211a678324ccfa39c10623c63f6aa 2024-12-11T20:35:33,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/d534a64c590e4ff8991c67a3748535ea is 50, key is test_row_0/B:col10/1733949331697/Put/seqid=0 2024-12-11T20:35:33,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742153_1329 (size=12301) 2024-12-11T20:35:33,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:33,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949393307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:33,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:33,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949393617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:33,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/d534a64c590e4ff8991c67a3748535ea 2024-12-11T20:35:33,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/2f70facf31254968bd305db931a8ad99 is 50, key is test_row_0/C:col10/1733949331697/Put/seqid=0 2024-12-11T20:35:33,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742154_1330 (size=12301) 2024-12-11T20:35:34,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/2f70facf31254968bd305db931a8ad99 2024-12-11T20:35:34,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/707211a678324ccfa39c10623c63f6aa as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/707211a678324ccfa39c10623c63f6aa 2024-12-11T20:35:34,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/707211a678324ccfa39c10623c63f6aa, entries=200, sequenceid=483, filesize=14.4 K 2024-12-11T20:35:34,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/d534a64c590e4ff8991c67a3748535ea as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d534a64c590e4ff8991c67a3748535ea 2024-12-11T20:35:34,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d534a64c590e4ff8991c67a3748535ea, entries=150, sequenceid=483, filesize=12.0 K 2024-12-11T20:35:34,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/2f70facf31254968bd305db931a8ad99 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2f70facf31254968bd305db931a8ad99 2024-12-11T20:35:34,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2f70facf31254968bd305db931a8ad99, entries=150, sequenceid=483, filesize=12.0 K 2024-12-11T20:35:34,125 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 320c74933b0c438c8808bab076a15dce in 1287ms, sequenceid=483, compaction requested=true 2024-12-11T20:35:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:34,125 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:34,125 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:34,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:34,126 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:34,126 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/A is initiating minor compaction (all files) 2024-12-11T20:35:34,126 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/A in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:34,126 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/2ca7939b69524c9aa3b2d374166c7290, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/dcd88ff0e09547878aa8dab7c0afb62c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/707211a678324ccfa39c10623c63f6aa] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=39.3 K 2024-12-11T20:35:34,126 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:34,126 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/B is initiating minor compaction (all files) 2024-12-11T20:35:34,126 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/B in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:34,126 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a2f7cbd414834621852d270b36234c7d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/8766b4d6c8b842a49cf224c5c972f9e7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d534a64c590e4ff8991c67a3748535ea] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.9 K 2024-12-11T20:35:34,126 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ca7939b69524c9aa3b2d374166c7290, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733949330334 2024-12-11T20:35:34,127 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting dcd88ff0e09547878aa8dab7c0afb62c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733949330526 2024-12-11T20:35:34,127 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f7cbd414834621852d270b36234c7d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733949330334 2024-12-11T20:35:34,127 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 707211a678324ccfa39c10623c63f6aa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949331690 2024-12-11T20:35:34,127 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 8766b4d6c8b842a49cf224c5c972f9e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733949330526 2024-12-11T20:35:34,127 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d534a64c590e4ff8991c67a3748535ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949331694 2024-12-11T20:35:34,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:34,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-11T20:35:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:34,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:34,134 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#A#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:34,135 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/5c2366e8133c4cd88ebe21ccba2aebef is 50, key is test_row_0/A:col10/1733949331697/Put/seqid=0 2024-12-11T20:35:34,135 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#B#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:34,135 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/9adc1367576b483c858cc4def575b3d3 is 50, key is test_row_0/B:col10/1733949331697/Put/seqid=0 2024-12-11T20:35:34,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/845110ddc2794540b877194f5d0fd806 is 50, key is test_row_0/A:col10/1733949332978/Put/seqid=0 2024-12-11T20:35:34,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742155_1331 (size=13255) 2024-12-11T20:35:34,148 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/5c2366e8133c4cd88ebe21ccba2aebef as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/5c2366e8133c4cd88ebe21ccba2aebef 2024-12-11T20:35:34,152 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/A of 320c74933b0c438c8808bab076a15dce into 5c2366e8133c4cd88ebe21ccba2aebef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:34,152 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:34,153 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/A, priority=13, startTime=1733949334125; duration=0sec 2024-12-11T20:35:34,153 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:34,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742156_1332 (size=13255) 2024-12-11T20:35:34,153 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:A 2024-12-11T20:35:34,153 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:34,154 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:34,155 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/C is initiating minor compaction (all files) 2024-12-11T20:35:34,155 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/C in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:34,155 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/63cc11b12226468594bcb47aafd01ac2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/095878b1b0184abfa601fc353d637da2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2f70facf31254968bd305db931a8ad99] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=36.9 K 2024-12-11T20:35:34,155 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63cc11b12226468594bcb47aafd01ac2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1733949330334 2024-12-11T20:35:34,155 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 095878b1b0184abfa601fc353d637da2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733949330526 2024-12-11T20:35:34,156 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f70facf31254968bd305db931a8ad99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949331694 2024-12-11T20:35:34,164 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#C#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:34,165 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/c60f540e52194a69abc9dd8a8db2050f is 50, key is test_row_0/C:col10/1733949331697/Put/seqid=0 2024-12-11T20:35:34,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742157_1333 (size=14741) 2024-12-11T20:35:34,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:34,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 344 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949394168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:34,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742158_1334 (size=13255) 2024-12-11T20:35:34,196 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/c60f540e52194a69abc9dd8a8db2050f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/c60f540e52194a69abc9dd8a8db2050f 2024-12-11T20:35:34,201 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/C of 320c74933b0c438c8808bab076a15dce into c60f540e52194a69abc9dd8a8db2050f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:34,201 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:34,201 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/C, priority=13, startTime=1733949334125; duration=0sec 2024-12-11T20:35:34,202 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:34,202 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:C 2024-12-11T20:35:34,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:34,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 345 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949394276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:34,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-11T20:35:34,467 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:34,467 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:34,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-11T20:35:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-11T20:35:34,469 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:34,470 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:34,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:34,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 346 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949394487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:34,557 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/9adc1367576b483c858cc4def575b3d3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/9adc1367576b483c858cc4def575b3d3 2024-12-11T20:35:34,561 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 320c74933b0c438c8808bab076a15dce/B of 320c74933b0c438c8808bab076a15dce into 9adc1367576b483c858cc4def575b3d3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:34,561 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:34,561 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/B, priority=13, startTime=1733949334125; duration=0sec 2024-12-11T20:35:34,561 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:34,561 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:B 2024-12-11T20:35:34,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/845110ddc2794540b877194f5d0fd806 2024-12-11T20:35:34,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/bc9cf2fc736b4ec0b0b4a92f1f79e3d1 is 50, key is test_row_0/B:col10/1733949332978/Put/seqid=0 2024-12-11T20:35:34,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-11T20:35:34,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742159_1335 (size=12301) 2024-12-11T20:35:34,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/bc9cf2fc736b4ec0b0b4a92f1f79e3d1 2024-12-11T20:35:34,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/0030c06b786f46e894173193c4c6fc15 is 50, key is test_row_0/C:col10/1733949332978/Put/seqid=0 2024-12-11T20:35:34,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742160_1336 (size=12301) 2024-12-11T20:35:34,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49266 deadline: 1733949394616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:34,620 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:34,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-11T20:35:34,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:34,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:34,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:34,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:34,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:34,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49298 deadline: 1733949394626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:34,629 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_0' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:34,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49268 deadline: 1733949394657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:34,660 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:34,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49322 deadline: 1733949394677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:34,680 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in 320c74933b0c438c8808bab076a15dce of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:35:34,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:34,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-11T20:35:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:34,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:34,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-11T20:35:34,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 347 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949394797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:34,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:34,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-11T20:35:34,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:34,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:34,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:34,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:34,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:34,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=520 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/0030c06b786f46e894173193c4c6fc15 2024-12-11T20:35:34,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/845110ddc2794540b877194f5d0fd806 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/845110ddc2794540b877194f5d0fd806 2024-12-11T20:35:34,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/845110ddc2794540b877194f5d0fd806, entries=200, sequenceid=520, filesize=14.4 K 2024-12-11T20:35:34,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/bc9cf2fc736b4ec0b0b4a92f1f79e3d1 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/bc9cf2fc736b4ec0b0b4a92f1f79e3d1 2024-12-11T20:35:34,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/bc9cf2fc736b4ec0b0b4a92f1f79e3d1, entries=150, sequenceid=520, filesize=12.0 K 2024-12-11T20:35:34,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/0030c06b786f46e894173193c4c6fc15 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/0030c06b786f46e894173193c4c6fc15 2024-12-11T20:35:35,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/0030c06b786f46e894173193c4c6fc15, entries=150, sequenceid=520, filesize=12.0 K 2024-12-11T20:35:35,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 320c74933b0c438c8808bab076a15dce in 874ms, sequenceid=520, compaction requested=false 2024-12-11T20:35:35,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:35,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-11T20:35:35,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:35,080 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:35:35,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:35,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:35,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:35,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:35,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:35,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:35,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/6c9ed4be12ae4627b82173413a5995e5 is 50, key is test_row_0/A:col10/1733949334131/Put/seqid=0 2024-12-11T20:35:35,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742161_1337 (size=12301) 2024-12-11T20:35:35,088 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/6c9ed4be12ae4627b82173413a5995e5 2024-12-11T20:35:35,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/e94a1f84f3f44af391695895ad1bcd14 is 50, key is test_row_0/B:col10/1733949334131/Put/seqid=0 2024-12-11T20:35:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-11T20:35:35,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742162_1338 (size=12301) 2024-12-11T20:35:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:35,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. as already flushing 2024-12-11T20:35:35,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1733949335345, max timestamp=1733949335367Num uniques 859; 2024-12-11T20:35:35,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1733949335345, max timestamp=1733949335367Num uniques 859; 2024-12-11T20:35:35,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=109, cellSize=4985, totalHeapSize=11576, min timestamp=1733949335345, max timestamp=1733949335367Num uniques 859; 2024-12-11T20:35:35,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-12-11T20:35:35,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-12-11T20:35:35,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=4985, suffix heap size=117064,new segment heap size=11576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=109 2024-12-11T20:35:35,460 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949335427, max timestamp=1733949335457Num uniques 968; 2024-12-11T20:35:35,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949335427, max timestamp=1733949335457Num uniques 968; 2024-12-11T20:35:35,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-12-11T20:35:35,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-12-11T20:35:35,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949335427, max timestamp=1733949335457Num uniques 968; 2024-12-11T20:35:35,461 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=44328, new segment data size=7688, suffix heap size=128640,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=968, new segment cells count=168 2024-12-11T20:35:35,499 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/e94a1f84f3f44af391695895ad1bcd14 2024-12-11T20:35:35,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/766b62ea97a241d5a75b4a7a50a3e85d is 50, key is test_row_0/C:col10/1733949334131/Put/seqid=0 2024-12-11T20:35:35,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742163_1339 (size=12301) 2024-12-11T20:35:35,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:35,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 399 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949395542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:35,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-11T20:35:35,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:35,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 400 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949395656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:35,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:35,789 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-12-11T20:35:35,790 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:35,790 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:35,790 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:35,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:35,790 INFO [Thread-1162 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1162. 2024-12-11T20:35:35,790 DEBUG [Thread-1162 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:35,790 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:35,790 DEBUG [Thread-1162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:35,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:35,791 INFO [Thread-1158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1158. 2024-12-11T20:35:35,791 DEBUG [Thread-1158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:35,791 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:35,791 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:35,791 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:35,792 INFO [Thread-1160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1160. 2024-12-11T20:35:35,792 DEBUG [Thread-1160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:35,792 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:35,792 DEBUG [Thread-1160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:35,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:35,793 INFO [Thread-1156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1156. 2024-12-11T20:35:35,793 DEBUG [Thread-1156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:35,793 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:35,793 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:35,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:35,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:35,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 401 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49310 deadline: 1733949395867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:35,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:35:35,926 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=533 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/766b62ea97a241d5a75b4a7a50a3e85d 2024-12-11T20:35:35,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/6c9ed4be12ae4627b82173413a5995e5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/6c9ed4be12ae4627b82173413a5995e5 2024-12-11T20:35:35,940 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/6c9ed4be12ae4627b82173413a5995e5, entries=150, sequenceid=533, filesize=12.0 K 2024-12-11T20:35:35,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/e94a1f84f3f44af391695895ad1bcd14 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/e94a1f84f3f44af391695895ad1bcd14 2024-12-11T20:35:35,944 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/e94a1f84f3f44af391695895ad1bcd14, entries=150, sequenceid=533, filesize=12.0 K 2024-12-11T20:35:35,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/766b62ea97a241d5a75b4a7a50a3e85d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/766b62ea97a241d5a75b4a7a50a3e85d 2024-12-11T20:35:35,948 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/766b62ea97a241d5a75b4a7a50a3e85d, entries=150, sequenceid=533, filesize=12.0 K 2024-12-11T20:35:35,949 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 320c74933b0c438c8808bab076a15dce in 869ms, sequenceid=533, compaction requested=true 2024-12-11T20:35:35,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:35,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:35,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-11T20:35:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-11T20:35:35,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-11T20:35:35,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4800 sec 2024-12-11T20:35:35,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.4850 sec 2024-12-11T20:35:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:36,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=140.89 KB heapSize=404.77 KB 2024-12-11T20:35:36,179 INFO [Thread-1143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1143. 2024-12-11T20:35:36,179 DEBUG [Thread-1143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:36,179 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:36,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:36,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T20:35:36,179 DEBUG [Thread-1143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:36,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:36,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T20:35:36,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:36,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T20:35:36,179 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:36,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/d32dfc005a6547dea59a0d19e4ff551e is 50, key is test_row_0/A:col10/1733949336177/Put/seqid=0 2024-12-11T20:35:36,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742164_1340 (size=12301) 2024-12-11T20:35:36,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/d32dfc005a6547dea59a0d19e4ff551e 2024-12-11T20:35:36,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/6e4ef437986b486ba2301d71b3d0f69d is 50, key is test_row_0/B:col10/1733949336177/Put/seqid=0 2024-12-11T20:35:36,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742165_1341 (size=12301) 2024-12-11T20:35:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-11T20:35:36,628 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:37,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/6e4ef437986b486ba2301d71b3d0f69d 2024-12-11T20:35:37,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/7460dc927abf4e64a680ad0a77abefb4 is 50, key is test_row_0/C:col10/1733949336177/Put/seqid=0 2024-12-11T20:35:37,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742166_1342 (size=12301) 2024-12-11T20:35:37,173 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T20:35:37,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=588 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/7460dc927abf4e64a680ad0a77abefb4 2024-12-11T20:35:37,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/d32dfc005a6547dea59a0d19e4ff551e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/d32dfc005a6547dea59a0d19e4ff551e 2024-12-11T20:35:37,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/d32dfc005a6547dea59a0d19e4ff551e, entries=150, sequenceid=588, filesize=12.0 K 2024-12-11T20:35:37,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/6e4ef437986b486ba2301d71b3d0f69d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6e4ef437986b486ba2301d71b3d0f69d 2024-12-11T20:35:37,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6e4ef437986b486ba2301d71b3d0f69d, entries=150, sequenceid=588, filesize=12.0 K 2024-12-11T20:35:37,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/7460dc927abf4e64a680ad0a77abefb4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/7460dc927abf4e64a680ad0a77abefb4 2024-12-11T20:35:37,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/7460dc927abf4e64a680ad0a77abefb4, entries=150, sequenceid=588, filesize=12.0 K 2024-12-11T20:35:37,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~404.72 KB/414432, currentSize=0 B/0 for 320c74933b0c438c8808bab076a15dce in 1275ms, sequenceid=588, compaction requested=true 2024-12-11T20:35:37,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:37,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:37,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:37,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:37,453 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:35:37,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:37,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 320c74933b0c438c8808bab076a15dce:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:37,453 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:35:37,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:37,455 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:35:37,455 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:35:37,455 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/A is initiating minor compaction (all files) 2024-12-11T20:35:37,455 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/B is initiating minor compaction (all files) 2024-12-11T20:35:37,455 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/A in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:37,455 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/B in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:37,455 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/5c2366e8133c4cd88ebe21ccba2aebef, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/845110ddc2794540b877194f5d0fd806, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/6c9ed4be12ae4627b82173413a5995e5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/d32dfc005a6547dea59a0d19e4ff551e] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=51.4 K 2024-12-11T20:35:37,455 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/9adc1367576b483c858cc4def575b3d3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/bc9cf2fc736b4ec0b0b4a92f1f79e3d1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/e94a1f84f3f44af391695895ad1bcd14, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6e4ef437986b486ba2301d71b3d0f69d] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=49.0 K 2024-12-11T20:35:37,456 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c2366e8133c4cd88ebe21ccba2aebef, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949331694 2024-12-11T20:35:37,456 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 9adc1367576b483c858cc4def575b3d3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949331694 2024-12-11T20:35:37,456 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting bc9cf2fc736b4ec0b0b4a92f1f79e3d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733949332967 2024-12-11T20:35:37,456 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 845110ddc2794540b877194f5d0fd806, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733949332967 2024-12-11T20:35:37,456 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting e94a1f84f3f44af391695895ad1bcd14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1733949334131 2024-12-11T20:35:37,456 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c9ed4be12ae4627b82173413a5995e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1733949334131 2024-12-11T20:35:37,457 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e4ef437986b486ba2301d71b3d0f69d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733949335522 2024-12-11T20:35:37,457 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32dfc005a6547dea59a0d19e4ff551e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733949335522 2024-12-11T20:35:37,467 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#A#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:37,467 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#B#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:37,468 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/e35d669de8e64aad9c4d4bfb3b7c0e9a is 50, key is test_row_0/A:col10/1733949336177/Put/seqid=0 2024-12-11T20:35:37,468 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/7ef885868fe24f69920b6a5b28a82b49 is 50, key is test_row_0/B:col10/1733949336177/Put/seqid=0 2024-12-11T20:35:37,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742168_1344 (size=13391) 2024-12-11T20:35:37,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742167_1343 (size=13391) 2024-12-11T20:35:37,905 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/7ef885868fe24f69920b6a5b28a82b49 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/7ef885868fe24f69920b6a5b28a82b49 2024-12-11T20:35:37,905 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/e35d669de8e64aad9c4d4bfb3b7c0e9a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e35d669de8e64aad9c4d4bfb3b7c0e9a 2024-12-11T20:35:37,910 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 320c74933b0c438c8808bab076a15dce/A of 320c74933b0c438c8808bab076a15dce into e35d669de8e64aad9c4d4bfb3b7c0e9a(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:37,910 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 320c74933b0c438c8808bab076a15dce/B of 320c74933b0c438c8808bab076a15dce into 7ef885868fe24f69920b6a5b28a82b49(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:37,910 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:37,910 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:37,910 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/A, priority=12, startTime=1733949337453; duration=0sec 2024-12-11T20:35:37,910 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/B, priority=12, startTime=1733949337453; duration=0sec 2024-12-11T20:35:37,910 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:37,910 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:B 2024-12-11T20:35:37,910 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:37,910 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:A 2024-12-11T20:35:37,910 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:35:37,911 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:35:37,911 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 320c74933b0c438c8808bab076a15dce/C is initiating minor compaction (all files) 2024-12-11T20:35:37,912 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 320c74933b0c438c8808bab076a15dce/C in TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:37,912 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/c60f540e52194a69abc9dd8a8db2050f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/0030c06b786f46e894173193c4c6fc15, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/766b62ea97a241d5a75b4a7a50a3e85d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/7460dc927abf4e64a680ad0a77abefb4] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp, totalSize=49.0 K 2024-12-11T20:35:37,912 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c60f540e52194a69abc9dd8a8db2050f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1733949331694 2024-12-11T20:35:37,912 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 0030c06b786f46e894173193c4c6fc15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=520, earliestPutTs=1733949332967 2024-12-11T20:35:37,912 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 766b62ea97a241d5a75b4a7a50a3e85d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=533, earliestPutTs=1733949334131 2024-12-11T20:35:37,913 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 7460dc927abf4e64a680ad0a77abefb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=588, earliestPutTs=1733949335522 2024-12-11T20:35:37,920 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 320c74933b0c438c8808bab076a15dce#C#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:37,920 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/9655f48e05f4496fba33391e8d209649 is 50, key is test_row_0/C:col10/1733949336177/Put/seqid=0 2024-12-11T20:35:37,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742169_1345 (size=13391) 2024-12-11T20:35:38,327 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/9655f48e05f4496fba33391e8d209649 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9655f48e05f4496fba33391e8d209649 2024-12-11T20:35:38,331 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 320c74933b0c438c8808bab076a15dce/C of 320c74933b0c438c8808bab076a15dce into 9655f48e05f4496fba33391e8d209649(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:38,331 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 320c74933b0c438c8808bab076a15dce: 2024-12-11T20:35:38,331 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., storeName=320c74933b0c438c8808bab076a15dce/C, priority=12, startTime=1733949337453; duration=0sec 2024-12-11T20:35:38,331 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:38,331 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 320c74933b0c438c8808bab076a15dce:C 2024-12-11T20:35:44,708 INFO [Thread-1147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1147. 2024-12-11T20:35:44,708 DEBUG [Thread-1147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:44,708 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:44,708 DEBUG [Thread-1147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:44,709 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:44,719 INFO [Thread-1145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1145. 2024-12-11T20:35:44,719 DEBUG [Thread-1145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:44,719 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:44,719 INFO [Thread-1151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1151. 2024-12-11T20:35:44,719 DEBUG [Thread-1151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:44,719 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:44,719 DEBUG [Thread-1145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:44,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:44,719 DEBUG [Thread-1151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:44,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:44,748 INFO [Thread-1149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1149. 2024-12-11T20:35:44,748 DEBUG [Thread-1149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:35:44,748 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:44,749 DEBUG [Thread-1149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:44,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 336 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4129 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4136 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4123 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4113 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4140 2024-12-11T20:35:44,749 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T20:35:44,749 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T20:35:44,749 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:35:44,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:44,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:44,749 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:35:44,750 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-11T20:35:44,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:44,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-11T20:35:44,753 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949344752"}]},"ts":"1733949344752"} 2024-12-11T20:35:44,754 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T20:35:44,754 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T20:35:44,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T20:35:44,756 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=320c74933b0c438c8808bab076a15dce, UNASSIGN}] 2024-12-11T20:35:44,757 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=320c74933b0c438c8808bab076a15dce, UNASSIGN 2024-12-11T20:35:44,757 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=320c74933b0c438c8808bab076a15dce, regionState=CLOSING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:44,759 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=320c74933b0c438c8808bab076a15dce, UNASSIGN because future has completed 2024-12-11T20:35:44,759 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T20:35:44,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:35:44,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-11T20:35:44,912 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:44,912 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T20:35:44,912 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 320c74933b0c438c8808bab076a15dce, disabling compactions & flushes 2024-12-11T20:35:44,912 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:44,912 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:44,912 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. after waiting 0 ms 2024-12-11T20:35:44,912 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:44,912 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 320c74933b0c438c8808bab076a15dce 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-11T20:35:44,912 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=A 2024-12-11T20:35:44,912 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:44,912 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=B 2024-12-11T20:35:44,913 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:44,913 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 320c74933b0c438c8808bab076a15dce, store=C 2024-12-11T20:35:44,913 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:44,916 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/e8f45b6335fe41738dc53e17a7f1b716 is 50, key is test_row_0/A:col10/1733949344718/Put/seqid=0 2024-12-11T20:35:44,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742170_1346 (size=9857) 2024-12-11T20:35:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-11T20:35:45,321 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/e8f45b6335fe41738dc53e17a7f1b716 2024-12-11T20:35:45,329 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/eb8fc9ba10694bbbb2364199a01e33bc is 50, key is test_row_0/B:col10/1733949344718/Put/seqid=0 2024-12-11T20:35:45,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742171_1347 (size=9857) 2024-12-11T20:35:45,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-11T20:35:45,738 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/eb8fc9ba10694bbbb2364199a01e33bc 2024-12-11T20:35:45,745 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/e7e00f0cc35041eba56ace2a1201b06d is 50, key is test_row_0/C:col10/1733949344718/Put/seqid=0 2024-12-11T20:35:45,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742172_1348 (size=9857) 2024-12-11T20:35:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-11T20:35:46,149 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/e7e00f0cc35041eba56ace2a1201b06d 2024-12-11T20:35:46,154 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/A/e8f45b6335fe41738dc53e17a7f1b716 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e8f45b6335fe41738dc53e17a7f1b716 2024-12-11T20:35:46,157 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e8f45b6335fe41738dc53e17a7f1b716, entries=100, sequenceid=598, filesize=9.6 K 2024-12-11T20:35:46,158 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/B/eb8fc9ba10694bbbb2364199a01e33bc as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/eb8fc9ba10694bbbb2364199a01e33bc 2024-12-11T20:35:46,162 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/eb8fc9ba10694bbbb2364199a01e33bc, entries=100, sequenceid=598, filesize=9.6 K 2024-12-11T20:35:46,163 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/.tmp/C/e7e00f0cc35041eba56ace2a1201b06d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/e7e00f0cc35041eba56ace2a1201b06d 2024-12-11T20:35:46,166 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/e7e00f0cc35041eba56ace2a1201b06d, entries=100, sequenceid=598, filesize=9.6 K 2024-12-11T20:35:46,167 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 320c74933b0c438c8808bab076a15dce in 1255ms, sequenceid=598, compaction requested=false 2024-12-11T20:35:46,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/3fd1bff5b9fc4917a40f835ca2ed0dfc, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/c5136573d6e8464f81eddbd088cfa486, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/567eee4c67a74d4d993d72e10984c248, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1075166efd8b47ac9106823bbbba550f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e3bb486bd0584124bb1f11081d23af84, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/7e2229a133454d048fefa9eddbf1e7a5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/9cbcb387c85d4505994b7ac2451becaa, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/877735eddc754244a3b5263171a4ee9c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1d33f08788514f8189719a224cd493d1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/61a8917ed21e426394bc9e9bbc6f05a0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/837713e05571499bae0c718843528f3e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e2e7e3dd80c44c78aeed02ab2c0720f5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/102bf94de6e842af8fa89378f7de6179, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/adbd0b0616bf45f7ae097a05c116cb4c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/afb481a6a3fa44bb83fee0da3a4971ee, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/a3ced0687628421bb85e7252455edae3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/ebd74b6defe845fea27e7b777852ce6e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/b987ba100243466387313b38df89b258, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bb1ffd15e65347bc90e7f8fa31d94033, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/fea069d035bb4577990005dba3f30430, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bc173ffd99604570a96b6b74046cb999, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/49350f5ec7b4464b8489caf17eae421f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/07eb82cb30a649f28f0ef8a6232bd253, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/434af51b7b8d423ea69719f46f2673d0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/2ca7939b69524c9aa3b2d374166c7290, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/84ccd53fdb9b4061b563a568289e2f7e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/dcd88ff0e09547878aa8dab7c0afb62c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/707211a678324ccfa39c10623c63f6aa, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/5c2366e8133c4cd88ebe21ccba2aebef, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/845110ddc2794540b877194f5d0fd806, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/6c9ed4be12ae4627b82173413a5995e5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/d32dfc005a6547dea59a0d19e4ff551e] to archive 2024-12-11T20:35:46,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:35:46,170 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/3fd1bff5b9fc4917a40f835ca2ed0dfc to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/3fd1bff5b9fc4917a40f835ca2ed0dfc 2024-12-11T20:35:46,170 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/c5136573d6e8464f81eddbd088cfa486 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/c5136573d6e8464f81eddbd088cfa486 2024-12-11T20:35:46,170 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1075166efd8b47ac9106823bbbba550f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1075166efd8b47ac9106823bbbba550f 2024-12-11T20:35:46,170 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/567eee4c67a74d4d993d72e10984c248 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/567eee4c67a74d4d993d72e10984c248 2024-12-11T20:35:46,170 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e3bb486bd0584124bb1f11081d23af84 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e3bb486bd0584124bb1f11081d23af84 2024-12-11T20:35:46,171 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/7e2229a133454d048fefa9eddbf1e7a5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/7e2229a133454d048fefa9eddbf1e7a5 2024-12-11T20:35:46,171 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/877735eddc754244a3b5263171a4ee9c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/877735eddc754244a3b5263171a4ee9c 2024-12-11T20:35:46,172 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1d33f08788514f8189719a224cd493d1 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/1d33f08788514f8189719a224cd493d1 2024-12-11T20:35:46,172 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/837713e05571499bae0c718843528f3e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/837713e05571499bae0c718843528f3e 2024-12-11T20:35:46,172 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e2e7e3dd80c44c78aeed02ab2c0720f5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e2e7e3dd80c44c78aeed02ab2c0720f5 2024-12-11T20:35:46,172 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/102bf94de6e842af8fa89378f7de6179 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/102bf94de6e842af8fa89378f7de6179 2024-12-11T20:35:46,172 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/61a8917ed21e426394bc9e9bbc6f05a0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/61a8917ed21e426394bc9e9bbc6f05a0 2024-12-11T20:35:46,172 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/9cbcb387c85d4505994b7ac2451becaa to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/9cbcb387c85d4505994b7ac2451becaa 2024-12-11T20:35:46,178 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/adbd0b0616bf45f7ae097a05c116cb4c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/adbd0b0616bf45f7ae097a05c116cb4c 2024-12-11T20:35:46,178 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/a3ced0687628421bb85e7252455edae3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/a3ced0687628421bb85e7252455edae3 2024-12-11T20:35:46,178 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/afb481a6a3fa44bb83fee0da3a4971ee to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/afb481a6a3fa44bb83fee0da3a4971ee 2024-12-11T20:35:46,178 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bb1ffd15e65347bc90e7f8fa31d94033 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bb1ffd15e65347bc90e7f8fa31d94033 2024-12-11T20:35:46,178 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/fea069d035bb4577990005dba3f30430 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/fea069d035bb4577990005dba3f30430 2024-12-11T20:35:46,179 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/ebd74b6defe845fea27e7b777852ce6e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/ebd74b6defe845fea27e7b777852ce6e 2024-12-11T20:35:46,179 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bc173ffd99604570a96b6b74046cb999 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/bc173ffd99604570a96b6b74046cb999 2024-12-11T20:35:46,179 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/b987ba100243466387313b38df89b258 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/b987ba100243466387313b38df89b258 2024-12-11T20:35:46,180 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/07eb82cb30a649f28f0ef8a6232bd253 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/07eb82cb30a649f28f0ef8a6232bd253 2024-12-11T20:35:46,180 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/49350f5ec7b4464b8489caf17eae421f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/49350f5ec7b4464b8489caf17eae421f 2024-12-11T20:35:46,180 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/84ccd53fdb9b4061b563a568289e2f7e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/84ccd53fdb9b4061b563a568289e2f7e 2024-12-11T20:35:46,180 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/dcd88ff0e09547878aa8dab7c0afb62c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/dcd88ff0e09547878aa8dab7c0afb62c 2024-12-11T20:35:46,180 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/2ca7939b69524c9aa3b2d374166c7290 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/2ca7939b69524c9aa3b2d374166c7290 2024-12-11T20:35:46,181 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/434af51b7b8d423ea69719f46f2673d0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/434af51b7b8d423ea69719f46f2673d0 2024-12-11T20:35:46,181 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/707211a678324ccfa39c10623c63f6aa to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/707211a678324ccfa39c10623c63f6aa 2024-12-11T20:35:46,181 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/5c2366e8133c4cd88ebe21ccba2aebef to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/5c2366e8133c4cd88ebe21ccba2aebef 2024-12-11T20:35:46,181 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/845110ddc2794540b877194f5d0fd806 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/845110ddc2794540b877194f5d0fd806 2024-12-11T20:35:46,182 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/d32dfc005a6547dea59a0d19e4ff551e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/d32dfc005a6547dea59a0d19e4ff551e 2024-12-11T20:35:46,182 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/6c9ed4be12ae4627b82173413a5995e5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/6c9ed4be12ae4627b82173413a5995e5 2024-12-11T20:35:46,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/c06cd746318b497b8d4c896a6deaeaf8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/39930a7752f44dc98199c41efbfa201e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/0ff2e44d05814e3dbdb785e1883a768d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/351a7d08f9624969918dad623f5cdf18, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/dd2e24a4029e4eae874dbc97d9337c02, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/2776acf147d2441890e963c7bfa97c8a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/cc231006ab1d4397a049ea9b708d0f71, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f9a49dbfe18d476696f478c3f2351d38, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1275246dfd0742fc938de721ae15b415, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/95bf69f31f3b402c807e52feb3355ceb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d600d020c8e04b5782ece5d65bdc158c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/9bb2985c495f4eb19fa6b563c6ca3ea3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a868c534a86e49238de9f6ef73e826c5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/78e7c9c5672747c097cd5fc9e487e999, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ebe906e9cbf476085757b70d27afa78, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/3e9486baf90446ebb8287acbffe7873a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/551ae87e8c6c4ad09d73fb34512c546d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/834f64e94c654cbaa1d5f8e10de9725b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/ec3c7d50e4af47e8bdf90a5be311fb78, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1e2448018252454ab41690aef9b70508, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/556d7254ab7a4553a9e1985bdfc57086, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f2effdb2f78846c786ba0da74fada3ba, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a189badeffac4d9287192f3c3ccadb6a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ce4ac96611d4ac1a1bd5e4bd9148650, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a2f7cbd414834621852d270b36234c7d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f133cbdd243145e88c4291a189d5d371, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/8766b4d6c8b842a49cf224c5c972f9e7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/9adc1367576b483c858cc4def575b3d3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d534a64c590e4ff8991c67a3748535ea, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/bc9cf2fc736b4ec0b0b4a92f1f79e3d1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/e94a1f84f3f44af391695895ad1bcd14, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6e4ef437986b486ba2301d71b3d0f69d] to archive 2024-12-11T20:35:46,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:35:46,186 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/c06cd746318b497b8d4c896a6deaeaf8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/c06cd746318b497b8d4c896a6deaeaf8 2024-12-11T20:35:46,186 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/0ff2e44d05814e3dbdb785e1883a768d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/0ff2e44d05814e3dbdb785e1883a768d 2024-12-11T20:35:46,186 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/39930a7752f44dc98199c41efbfa201e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/39930a7752f44dc98199c41efbfa201e 2024-12-11T20:35:46,187 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/351a7d08f9624969918dad623f5cdf18 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/351a7d08f9624969918dad623f5cdf18 2024-12-11T20:35:46,187 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/dd2e24a4029e4eae874dbc97d9337c02 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/dd2e24a4029e4eae874dbc97d9337c02 2024-12-11T20:35:46,188 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/2776acf147d2441890e963c7bfa97c8a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/2776acf147d2441890e963c7bfa97c8a 2024-12-11T20:35:46,188 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f9a49dbfe18d476696f478c3f2351d38 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f9a49dbfe18d476696f478c3f2351d38 2024-12-11T20:35:46,188 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/cc231006ab1d4397a049ea9b708d0f71 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/cc231006ab1d4397a049ea9b708d0f71 2024-12-11T20:35:46,188 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1275246dfd0742fc938de721ae15b415 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1275246dfd0742fc938de721ae15b415 2024-12-11T20:35:46,189 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/95bf69f31f3b402c807e52feb3355ceb to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/95bf69f31f3b402c807e52feb3355ceb 2024-12-11T20:35:46,189 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d600d020c8e04b5782ece5d65bdc158c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d600d020c8e04b5782ece5d65bdc158c 2024-12-11T20:35:46,189 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/9bb2985c495f4eb19fa6b563c6ca3ea3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/9bb2985c495f4eb19fa6b563c6ca3ea3 2024-12-11T20:35:46,190 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a868c534a86e49238de9f6ef73e826c5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a868c534a86e49238de9f6ef73e826c5 2024-12-11T20:35:46,190 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/78e7c9c5672747c097cd5fc9e487e999 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/78e7c9c5672747c097cd5fc9e487e999 2024-12-11T20:35:46,190 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/3e9486baf90446ebb8287acbffe7873a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/3e9486baf90446ebb8287acbffe7873a 2024-12-11T20:35:46,190 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ebe906e9cbf476085757b70d27afa78 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ebe906e9cbf476085757b70d27afa78 2024-12-11T20:35:46,190 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/ec3c7d50e4af47e8bdf90a5be311fb78 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/ec3c7d50e4af47e8bdf90a5be311fb78 2024-12-11T20:35:46,190 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/834f64e94c654cbaa1d5f8e10de9725b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/834f64e94c654cbaa1d5f8e10de9725b 2024-12-11T20:35:46,192 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1e2448018252454ab41690aef9b70508 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/1e2448018252454ab41690aef9b70508 2024-12-11T20:35:46,192 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/551ae87e8c6c4ad09d73fb34512c546d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/551ae87e8c6c4ad09d73fb34512c546d 2024-12-11T20:35:46,192 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/556d7254ab7a4553a9e1985bdfc57086 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/556d7254ab7a4553a9e1985bdfc57086 2024-12-11T20:35:46,193 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f2effdb2f78846c786ba0da74fada3ba to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f2effdb2f78846c786ba0da74fada3ba 2024-12-11T20:35:46,193 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a189badeffac4d9287192f3c3ccadb6a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a189badeffac4d9287192f3c3ccadb6a 2024-12-11T20:35:46,193 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a2f7cbd414834621852d270b36234c7d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/a2f7cbd414834621852d270b36234c7d 2024-12-11T20:35:46,193 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ce4ac96611d4ac1a1bd5e4bd9148650 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6ce4ac96611d4ac1a1bd5e4bd9148650 2024-12-11T20:35:46,193 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f133cbdd243145e88c4291a189d5d371 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/f133cbdd243145e88c4291a189d5d371 2024-12-11T20:35:46,193 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/8766b4d6c8b842a49cf224c5c972f9e7 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/8766b4d6c8b842a49cf224c5c972f9e7 2024-12-11T20:35:46,194 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/9adc1367576b483c858cc4def575b3d3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/9adc1367576b483c858cc4def575b3d3 2024-12-11T20:35:46,194 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d534a64c590e4ff8991c67a3748535ea to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/d534a64c590e4ff8991c67a3748535ea 2024-12-11T20:35:46,194 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/bc9cf2fc736b4ec0b0b4a92f1f79e3d1 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/bc9cf2fc736b4ec0b0b4a92f1f79e3d1 2024-12-11T20:35:46,194 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6e4ef437986b486ba2301d71b3d0f69d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/6e4ef437986b486ba2301d71b3d0f69d 2024-12-11T20:35:46,194 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/e94a1f84f3f44af391695895ad1bcd14 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/e94a1f84f3f44af391695895ad1bcd14 2024-12-11T20:35:46,195 DEBUG [StoreCloser-TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3e25923833d0445583ddba3fee3d41e4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/55c1df3968794321aa664fe7ac314e76, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3be7e62335e34effa39a1312f14204d5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/02354fdd56e04eabbbaa0c6cb7c1326d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/d2267dcb1381474fba933529ea01fb21, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/7984b6d344e547d181a96d4943e6336b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/f2a03473fb0f4ed6a3189db83378e272, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/1101f4f214e74cfc95fab108f209709d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/4974f89c5ec24627b212176eb42e355b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/efd5a3e2fd9946929e1dc5efb29510ef, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9841c362fbd145ebb2d85b515e35986e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9dd3b152b1c34cdc8f38cf61c047cce1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/61b448791a644c7781de4cb091db8079, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/85d7fd1b476d442c850e006d19ae95b1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/b6c9f927d3534fb9925d9ddf4cc53b85, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/e9a0f6ce07884c70a27396dca6a1728e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2a343ab4d34f4af1b886189a9edba715, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/fc353fac98ed4aaf943ac523471044e5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/0bc729a206ee484ab6a77fed489ef8f5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/c0edc306d5f74ea7bfe5e7d812815d4d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6f719f77aea04a44b50d1b32d9bff1e5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/f0700fa86c6048b3931cf8311e45f9b6, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6e9a66610e684efd91c1025dfb5c78e5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3c53ec4ec1804ba18ca17f5049cd76f6, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/63cc11b12226468594bcb47aafd01ac2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/058a5a7a696c49a397c0352f1a1f137a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/095878b1b0184abfa601fc353d637da2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/c60f540e52194a69abc9dd8a8db2050f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2f70facf31254968bd305db931a8ad99, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/0030c06b786f46e894173193c4c6fc15, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/766b62ea97a241d5a75b4a7a50a3e85d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/7460dc927abf4e64a680ad0a77abefb4] to archive 2024-12-11T20:35:46,196 DEBUG [StoreCloser-TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:35:46,197 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3e25923833d0445583ddba3fee3d41e4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3e25923833d0445583ddba3fee3d41e4 2024-12-11T20:35:46,197 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3be7e62335e34effa39a1312f14204d5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3be7e62335e34effa39a1312f14204d5 2024-12-11T20:35:46,197 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/f2a03473fb0f4ed6a3189db83378e272 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/f2a03473fb0f4ed6a3189db83378e272 2024-12-11T20:35:46,198 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/55c1df3968794321aa664fe7ac314e76 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/55c1df3968794321aa664fe7ac314e76 2024-12-11T20:35:46,198 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/1101f4f214e74cfc95fab108f209709d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/1101f4f214e74cfc95fab108f209709d 2024-12-11T20:35:46,198 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/02354fdd56e04eabbbaa0c6cb7c1326d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/02354fdd56e04eabbbaa0c6cb7c1326d 2024-12-11T20:35:46,198 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/7984b6d344e547d181a96d4943e6336b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/7984b6d344e547d181a96d4943e6336b 2024-12-11T20:35:46,199 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/d2267dcb1381474fba933529ea01fb21 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/d2267dcb1381474fba933529ea01fb21 2024-12-11T20:35:46,199 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/efd5a3e2fd9946929e1dc5efb29510ef to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/efd5a3e2fd9946929e1dc5efb29510ef 2024-12-11T20:35:46,200 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9841c362fbd145ebb2d85b515e35986e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9841c362fbd145ebb2d85b515e35986e 2024-12-11T20:35:46,200 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9dd3b152b1c34cdc8f38cf61c047cce1 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9dd3b152b1c34cdc8f38cf61c047cce1 2024-12-11T20:35:46,201 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/61b448791a644c7781de4cb091db8079 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/61b448791a644c7781de4cb091db8079 2024-12-11T20:35:46,201 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2a343ab4d34f4af1b886189a9edba715 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2a343ab4d34f4af1b886189a9edba715 2024-12-11T20:35:46,201 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/4974f89c5ec24627b212176eb42e355b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/4974f89c5ec24627b212176eb42e355b 2024-12-11T20:35:46,201 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/e9a0f6ce07884c70a27396dca6a1728e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/e9a0f6ce07884c70a27396dca6a1728e 2024-12-11T20:35:46,201 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/85d7fd1b476d442c850e006d19ae95b1 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/85d7fd1b476d442c850e006d19ae95b1 2024-12-11T20:35:46,202 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/0bc729a206ee484ab6a77fed489ef8f5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/0bc729a206ee484ab6a77fed489ef8f5 2024-12-11T20:35:46,202 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/fc353fac98ed4aaf943ac523471044e5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/fc353fac98ed4aaf943ac523471044e5 2024-12-11T20:35:46,202 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/c0edc306d5f74ea7bfe5e7d812815d4d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/c0edc306d5f74ea7bfe5e7d812815d4d 2024-12-11T20:35:46,203 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/b6c9f927d3534fb9925d9ddf4cc53b85 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/b6c9f927d3534fb9925d9ddf4cc53b85 2024-12-11T20:35:46,203 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/f0700fa86c6048b3931cf8311e45f9b6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/f0700fa86c6048b3931cf8311e45f9b6 2024-12-11T20:35:46,203 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6e9a66610e684efd91c1025dfb5c78e5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6e9a66610e684efd91c1025dfb5c78e5 2024-12-11T20:35:46,203 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6f719f77aea04a44b50d1b32d9bff1e5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/6f719f77aea04a44b50d1b32d9bff1e5 2024-12-11T20:35:46,204 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3c53ec4ec1804ba18ca17f5049cd76f6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/3c53ec4ec1804ba18ca17f5049cd76f6 2024-12-11T20:35:46,204 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/058a5a7a696c49a397c0352f1a1f137a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/058a5a7a696c49a397c0352f1a1f137a 2024-12-11T20:35:46,204 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/63cc11b12226468594bcb47aafd01ac2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/63cc11b12226468594bcb47aafd01ac2 2024-12-11T20:35:46,205 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/766b62ea97a241d5a75b4a7a50a3e85d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/766b62ea97a241d5a75b4a7a50a3e85d 2024-12-11T20:35:46,205 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/095878b1b0184abfa601fc353d637da2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/095878b1b0184abfa601fc353d637da2 2024-12-11T20:35:46,205 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/0030c06b786f46e894173193c4c6fc15 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/0030c06b786f46e894173193c4c6fc15 2024-12-11T20:35:46,206 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2f70facf31254968bd305db931a8ad99 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/2f70facf31254968bd305db931a8ad99 2024-12-11T20:35:46,206 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/c60f540e52194a69abc9dd8a8db2050f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/c60f540e52194a69abc9dd8a8db2050f 2024-12-11T20:35:46,206 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/7460dc927abf4e64a680ad0a77abefb4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/7460dc927abf4e64a680ad0a77abefb4 2024-12-11T20:35:46,211 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/recovered.edits/601.seqid, newMaxSeqId=601, maxSeqId=1 2024-12-11T20:35:46,212 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. 2024-12-11T20:35:46,212 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 320c74933b0c438c8808bab076a15dce: Waiting for close lock at 1733949344912Running coprocessor pre-close hooks at 1733949344912Disabling compacts and flushes for region at 1733949344912Disabling writes for close at 1733949344912Obtaining lock to block concurrent updates at 1733949344912Preparing flush snapshotting stores in 320c74933b0c438c8808bab076a15dce at 1733949344912Finished memstore snapshotting TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733949344913 (+1 ms)Flushing stores of TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce. at 1733949344913Flushing 320c74933b0c438c8808bab076a15dce/A: creating writer at 1733949344913Flushing 320c74933b0c438c8808bab076a15dce/A: appending metadata at 1733949344916 (+3 ms)Flushing 320c74933b0c438c8808bab076a15dce/A: closing flushed file at 1733949344916Flushing 320c74933b0c438c8808bab076a15dce/B: creating writer at 1733949345326 (+410 ms)Flushing 320c74933b0c438c8808bab076a15dce/B: appending metadata at 1733949345328 (+2 ms)Flushing 320c74933b0c438c8808bab076a15dce/B: closing flushed file at 1733949345329 (+1 ms)Flushing 320c74933b0c438c8808bab076a15dce/C: creating writer at 1733949345742 (+413 ms)Flushing 320c74933b0c438c8808bab076a15dce/C: appending metadata at 1733949345744 (+2 ms)Flushing 320c74933b0c438c8808bab076a15dce/C: closing flushed file at 1733949345744Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fbcc847: reopening flushed file at 1733949346153 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@170aad8c: reopening flushed file at 1733949346158 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fa882d4: reopening flushed file at 1733949346162 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 320c74933b0c438c8808bab076a15dce in 1255ms, sequenceid=598, compaction requested=false at 1733949346167 (+5 ms)Writing region close event to WAL at 1733949346208 (+41 ms)Running coprocessor post-close hooks at 1733949346212 (+4 ms)Closed at 1733949346212 2024-12-11T20:35:46,214 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:46,214 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=320c74933b0c438c8808bab076a15dce, regionState=CLOSED 2024-12-11T20:35:46,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:35:46,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-11T20:35:46,221 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 320c74933b0c438c8808bab076a15dce, server=5be53b084ac7,44543,1733949248752 in 1.4610 sec 2024-12-11T20:35:46,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-11T20:35:46,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=320c74933b0c438c8808bab076a15dce, UNASSIGN in 1.4650 sec 2024-12-11T20:35:46,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-11T20:35:46,227 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4710 sec 2024-12-11T20:35:46,228 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949346228"}]},"ts":"1733949346228"} 2024-12-11T20:35:46,229 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T20:35:46,229 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T20:35:46,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4800 sec 2024-12-11T20:35:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-11T20:35:46,898 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:46,898 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-11T20:35:46,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:46,900 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:46,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-11T20:35:46,901 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:46,903 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:46,905 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/recovered.edits] 2024-12-11T20:35:46,907 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e35d669de8e64aad9c4d4bfb3b7c0e9a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e35d669de8e64aad9c4d4bfb3b7c0e9a 2024-12-11T20:35:46,907 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e8f45b6335fe41738dc53e17a7f1b716 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/A/e8f45b6335fe41738dc53e17a7f1b716 2024-12-11T20:35:46,909 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/eb8fc9ba10694bbbb2364199a01e33bc to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/eb8fc9ba10694bbbb2364199a01e33bc 2024-12-11T20:35:46,909 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/7ef885868fe24f69920b6a5b28a82b49 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/B/7ef885868fe24f69920b6a5b28a82b49 2024-12-11T20:35:46,911 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9655f48e05f4496fba33391e8d209649 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/9655f48e05f4496fba33391e8d209649 2024-12-11T20:35:46,912 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/e7e00f0cc35041eba56ace2a1201b06d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/C/e7e00f0cc35041eba56ace2a1201b06d 2024-12-11T20:35:46,914 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/recovered.edits/601.seqid to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce/recovered.edits/601.seqid 2024-12-11T20:35:46,914 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/320c74933b0c438c8808bab076a15dce 2024-12-11T20:35:46,914 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T20:35:46,916 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:46,919 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T20:35:46,921 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T20:35:46,923 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:46,923 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T20:35:46,923 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733949346923"}]},"ts":"9223372036854775807"} 2024-12-11T20:35:46,926 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T20:35:46,926 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 320c74933b0c438c8808bab076a15dce, NAME => 'TestAcidGuarantees,,1733949313526.320c74933b0c438c8808bab076a15dce.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T20:35:46,926 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T20:35:46,926 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733949346926"}]},"ts":"9223372036854775807"} 2024-12-11T20:35:46,929 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T20:35:46,930 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:46,931 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-12-11T20:35:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-12-11T20:35:47,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T20:35:47,007 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:47,018 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=253 (was 255), OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=397 (was 484), ProcessCount=11 (was 11), AvailableMemoryMB=1712 (was 1890) 2024-12-11T20:35:47,027 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=253, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=397, ProcessCount=11, AvailableMemoryMB=1711 2024-12-11T20:35:47,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T20:35:47,028 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:35:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:47,030 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:35:47,031 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:47,031 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-12-11T20:35:47,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-11T20:35:47,032 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:35:47,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742173_1349 (size=1061) 2024-12-11T20:35:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-11T20:35:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-11T20:35:47,439 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043 2024-12-11T20:35:47,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742174_1350 (size=53) 2024-12-11T20:35:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-11T20:35:47,844 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:35:47,844 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4451ee260f37d1215790a309e039ba65, disabling compactions & flushes 2024-12-11T20:35:47,844 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:47,844 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:47,844 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. after waiting 0 ms 2024-12-11T20:35:47,844 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:47,844 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:47,844 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4451ee260f37d1215790a309e039ba65: Waiting for close lock at 1733949347844Disabling compacts and flushes for region at 1733949347844Disabling writes for close at 1733949347844Writing region close event to WAL at 1733949347844Closed at 1733949347844 2024-12-11T20:35:47,846 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:35:47,846 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733949347846"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733949347846"}]},"ts":"1733949347846"} 2024-12-11T20:35:47,848 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:35:47,849 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:35:47,849 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949347849"}]},"ts":"1733949347849"} 2024-12-11T20:35:47,850 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T20:35:47,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, ASSIGN}] 2024-12-11T20:35:47,852 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, ASSIGN 2024-12-11T20:35:47,852 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, ASSIGN; state=OFFLINE, location=5be53b084ac7,44543,1733949248752; forceNewPlan=false, retain=false 2024-12-11T20:35:48,003 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=4451ee260f37d1215790a309e039ba65, regionState=OPENING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:48,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, ASSIGN because future has completed 2024-12-11T20:35:48,005 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:35:48,159 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:48,159 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:35:48,160 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,160 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:35:48,160 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,160 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,161 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,162 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:35:48,162 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4451ee260f37d1215790a309e039ba65 columnFamilyName A 2024-12-11T20:35:48,162 DEBUG [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:48,162 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(327): Store=4451ee260f37d1215790a309e039ba65/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:35:48,162 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,163 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:35:48,163 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4451ee260f37d1215790a309e039ba65 columnFamilyName B 2024-12-11T20:35:48,163 DEBUG [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:48,164 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(327): Store=4451ee260f37d1215790a309e039ba65/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:35:48,164 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,164 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:35:48,165 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4451ee260f37d1215790a309e039ba65 columnFamilyName C 2024-12-11T20:35:48,165 DEBUG [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:48,165 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(327): Store=4451ee260f37d1215790a309e039ba65/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:35:48,165 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:48,165 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,166 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,166 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,167 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,167 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,167 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T20:35:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-11T20:35:48,168 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,169 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:35:48,170 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 4451ee260f37d1215790a309e039ba65; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71740173, jitterRate=0.0690118819475174}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T20:35:48,170 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:48,170 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 4451ee260f37d1215790a309e039ba65: Running coprocessor pre-open hook at 1733949348160Writing region info on filesystem at 1733949348160Initializing all the Stores at 1733949348160Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949348161 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949348161Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949348161Cleaning up temporary data from old regions at 1733949348167 (+6 ms)Running coprocessor post-open hooks at 1733949348170 (+3 ms)Region opened successfully at 1733949348170 2024-12-11T20:35:48,171 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., pid=91, masterSystemTime=1733949348157 2024-12-11T20:35:48,172 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:48,172 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:48,172 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=4451ee260f37d1215790a309e039ba65, regionState=OPEN, openSeqNum=2, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:48,173 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:35:48,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-12-11T20:35:48,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 in 169 msec 2024-12-11T20:35:48,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-11T20:35:48,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, ASSIGN in 325 msec 2024-12-11T20:35:48,178 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:35:48,178 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949348178"}]},"ts":"1733949348178"} 2024-12-11T20:35:48,179 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T20:35:48,180 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:35:48,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-12-11T20:35:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-11T20:35:49,177 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:49,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65cae09d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:49,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:49,178 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:49,179 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749ba631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:49,180 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59898, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:49,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2a23bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:49,181 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:49,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:49,184 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T20:35:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 5be53b084ac7,39923,1733949248041 2024-12-11T20:35:49,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1d5cb2a 2024-12-11T20:35:49,185 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T20:35:49,186 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T20:35:49,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T20:35:49,187 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:35:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-11T20:35:49,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742175_1351 (size=1097) 2024-12-11T20:35:49,601 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-11T20:35:49,601 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-11T20:35:49,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T20:35:49,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, REOPEN/MOVE}] 2024-12-11T20:35:49,610 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, REOPEN/MOVE 2024-12-11T20:35:49,611 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=4451ee260f37d1215790a309e039ba65, regionState=CLOSING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:49,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, REOPEN/MOVE because future has completed 2024-12-11T20:35:49,613 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T20:35:49,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:35:49,767 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:49,767 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T20:35:49,767 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 4451ee260f37d1215790a309e039ba65, disabling compactions & flushes 2024-12-11T20:35:49,767 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:49,767 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:49,767 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. after waiting 0 ms 2024-12-11T20:35:49,767 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:49,772 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T20:35:49,773 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:49,773 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 4451ee260f37d1215790a309e039ba65: Waiting for close lock at 1733949349767Running coprocessor pre-close hooks at 1733949349767Disabling compacts and flushes for region at 1733949349767Disabling writes for close at 1733949349767Writing region close event to WAL at 1733949349768 (+1 ms)Running coprocessor post-close hooks at 1733949349773 (+5 ms)Closed at 1733949349773 2024-12-11T20:35:49,773 WARN [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 4451ee260f37d1215790a309e039ba65 to self. 2024-12-11T20:35:49,776 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:49,776 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=4451ee260f37d1215790a309e039ba65, regionState=CLOSED 2024-12-11T20:35:49,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:35:49,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-11T20:35:49,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 in 166 msec 2024-12-11T20:35:49,781 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, REOPEN/MOVE; state=CLOSED, location=5be53b084ac7,44543,1733949248752; forceNewPlan=false, retain=true 2024-12-11T20:35:49,932 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=4451ee260f37d1215790a309e039ba65, regionState=OPENING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:49,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, REOPEN/MOVE because future has completed 2024-12-11T20:35:49,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:35:50,094 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,095 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:35:50,096 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,096 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:35:50,096 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,096 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,099 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,100 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:35:50,100 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4451ee260f37d1215790a309e039ba65 columnFamilyName A 2024-12-11T20:35:50,102 DEBUG [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:50,102 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(327): Store=4451ee260f37d1215790a309e039ba65/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:35:50,103 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,103 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:35:50,103 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4451ee260f37d1215790a309e039ba65 columnFamilyName B 2024-12-11T20:35:50,103 DEBUG [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:50,104 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(327): Store=4451ee260f37d1215790a309e039ba65/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:35:50,104 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,104 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:35:50,104 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4451ee260f37d1215790a309e039ba65 columnFamilyName C 2024-12-11T20:35:50,104 DEBUG [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:50,104 INFO [StoreOpener-4451ee260f37d1215790a309e039ba65-1 {}] regionserver.HStore(327): Store=4451ee260f37d1215790a309e039ba65/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:35:50,104 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,105 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,105 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,106 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,107 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,107 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,107 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T20:35:50,108 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,109 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 4451ee260f37d1215790a309e039ba65; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67404517, jitterRate=0.004405573010444641}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T20:35:50,109 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,110 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 4451ee260f37d1215790a309e039ba65: Running coprocessor pre-open hook at 1733949350097Writing region info on filesystem at 1733949350097Initializing all the Stores at 1733949350098 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949350098Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949350099 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949350099Cleaning up temporary data from old regions at 1733949350107 (+8 ms)Running coprocessor post-open hooks at 1733949350109 (+2 ms)Region opened successfully at 1733949350110 (+1 ms) 2024-12-11T20:35:50,111 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., pid=96, masterSystemTime=1733949350088 2024-12-11T20:35:50,112 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,112 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,113 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=4451ee260f37d1215790a309e039ba65, regionState=OPEN, openSeqNum=5, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:35:50,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-12-11T20:35:50,116 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 in 179 msec 2024-12-11T20:35:50,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-11T20:35:50,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, REOPEN/MOVE in 507 msec 2024-12-11T20:35:50,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-11T20:35:50,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 512 msec 2024-12-11T20:35:50,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 931 msec 2024-12-11T20:35:50,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-11T20:35:50,121 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:50,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@408114f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,122 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:50,122 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:50,123 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2792f786, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:50,124 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194b44b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fcc3737, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,125 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:50,125 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:50,126 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bdceec1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:50,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:50,127 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:50,127 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@627c508d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47b5d3d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:50,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:50,128 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:50,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@660f1986, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:50,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:50,129 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:50,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bef92de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,130 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e9f115, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:50,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:50,131 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636e392f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:50,132 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:50,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b85284, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69fe1579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,133 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:50,133 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:50,133 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cbc9be9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:50,134 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121ae9a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e9b8463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:50,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:50,136 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e285d25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:50,137 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60024, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ed92bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6821edd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,137 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:50,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:50,138 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@227b3c4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:50,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:50,139 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596abadb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42981879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:50,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:50,141 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37795e24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:50,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:50,142 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6918383d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736bcb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:50,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:50,144 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22cd3155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:50,145 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef37fa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74035f7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:35:50,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:35:50,147 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:35:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:35:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:35:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48ca9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:35:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:35:50,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:35:50,148 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:35:50,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e00f2fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:35:50,149 DEBUG [Thread-1611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:50,149 DEBUG [Thread-1615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:50,149 DEBUG [Thread-1617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:50,149 DEBUG [Thread-1613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:50,149 DEBUG [Thread-1600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:50,150 DEBUG [Thread-1598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:50,150 DEBUG [Thread-1602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:50,150 DEBUG [Thread-1606 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:50,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:50,151 DEBUG [Thread-1604 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:50,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:50,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:50,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:50,151 DEBUG [Thread-1600 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:50,151 DEBUG [Thread-1598 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:50,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:50,151 DEBUG [Thread-1600 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:50,151 DEBUG [Thread-1598 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:50,151 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:50,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:50,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:50,152 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:50,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:50,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:50,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:50,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:50,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:50,154 DEBUG [Thread-1609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:35:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:35:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:35:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:35:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:35:50,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:35:50,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:35:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:35:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:35:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:35:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-11T20:35:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:35:50,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:35:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T20:35:50,160 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:35:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:35:50,161 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:50,161 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:50,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:35:50,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:35:50,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:50,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:35:50,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:50,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:35:50,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:50,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211710b528ea0ef4a3db208cb74ad90eb1c_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949350188/Put/seqid=0 2024-12-11T20:35:50,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733949350206, max timestamp=1733949350221Num uniques 859; 2024-12-11T20:35:50,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T20:35:50,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733949350206, max timestamp=1733949350221Num uniques 859; 2024-12-11T20:35:50,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733949350206, max timestamp=1733949350221Num uniques 859; 2024-12-11T20:35:50,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T20:35:50,224 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T20:35:50,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742176_1352 (size=12154) 2024-12-11T20:35:50,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:50,239 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211710b528ea0ef4a3db208cb74ad90eb1c_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211710b528ea0ef4a3db208cb74ad90eb1c_4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:50,239 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/128d6c41763b4459adae166b279e0f7f, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:50,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/128d6c41763b4459adae166b279e0f7f is 175, key is test_row_0/A:col10/1733949350188/Put/seqid=0 2024-12-11T20:35:50,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742177_1353 (size=30955) 2024-12-11T20:35:50,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949410239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949410245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949410246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949410248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949410249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T20:35:50,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T20:35:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949410357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949410357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949410357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949410357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949410367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T20:35:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T20:35:50,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949410567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949410567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949410567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949410577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949410577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T20:35:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,644 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/128d6c41763b4459adae166b279e0f7f 2024-12-11T20:35:50,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/e2b1b09406c646eb8c38f9848a0924f7 is 50, key is test_row_0/B:col10/1733949350188/Put/seqid=0 2024-12-11T20:35:50,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742178_1354 (size=12001) 2024-12-11T20:35:50,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/e2b1b09406c646eb8c38f9848a0924f7 2024-12-11T20:35:50,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/6f3c52fc7beb4a7b8890f3a770e39e1b is 50, key is test_row_0/C:col10/1733949350188/Put/seqid=0 2024-12-11T20:35:50,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742179_1355 (size=12001) 2024-12-11T20:35:50,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T20:35:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T20:35:50,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949410877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949410877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949410877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949410887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949410887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:50,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T20:35:50,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:50,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:50,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:51,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T20:35:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:51,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:51,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:51,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/6f3c52fc7beb4a7b8890f3a770e39e1b 2024-12-11T20:35:51,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/128d6c41763b4459adae166b279e0f7f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/128d6c41763b4459adae166b279e0f7f 2024-12-11T20:35:51,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/128d6c41763b4459adae166b279e0f7f, entries=150, sequenceid=14, filesize=30.2 K 2024-12-11T20:35:51,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/e2b1b09406c646eb8c38f9848a0924f7 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e2b1b09406c646eb8c38f9848a0924f7 2024-12-11T20:35:51,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e2b1b09406c646eb8c38f9848a0924f7, entries=150, sequenceid=14, filesize=11.7 K 2024-12-11T20:35:51,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/6f3c52fc7beb4a7b8890f3a770e39e1b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6f3c52fc7beb4a7b8890f3a770e39e1b 2024-12-11T20:35:51,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6f3c52fc7beb4a7b8890f3a770e39e1b, entries=150, sequenceid=14, filesize=11.7 K 2024-12-11T20:35:51,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4451ee260f37d1215790a309e039ba65 in 976ms, sequenceid=14, compaction requested=false 2024-12-11T20:35:51,166 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-11T20:35:51,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:51,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-11T20:35:51,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:51,235 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-12-11T20:35:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:35:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:35:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:35:51,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:35:51,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bf2291fbd710468da3bcaa09393a9495_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949350237/Put/seqid=0 2024-12-11T20:35:51,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742180_1356 (size=12154) 2024-12-11T20:35:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T20:35:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:51,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:51,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949411411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949411411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949411412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949411414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949411414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949411517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949411527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949411527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949411527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949411527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:51,665 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bf2291fbd710468da3bcaa09393a9495_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bf2291fbd710468da3bcaa09393a9495_4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:51,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/512d51a2a7154ee49032c08fc4eeea24, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:51,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/512d51a2a7154ee49032c08fc4eeea24 is 175, key is test_row_0/A:col10/1733949350237/Put/seqid=0 2024-12-11T20:35:51,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742181_1357 (size=30955) 2024-12-11T20:35:51,741 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:35:51,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949411737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949411737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949411737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949411737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949411738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:51,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949412047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949412047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:52,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:52,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949412047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:52,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:52,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949412057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:52,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949412058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:52,073 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/512d51a2a7154ee49032c08fc4eeea24 2024-12-11T20:35:52,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/0e10cfdbeead4c4ba9660df4943720d0 is 50, key is test_row_0/B:col10/1733949350237/Put/seqid=0 2024-12-11T20:35:52,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742182_1358 (size=12001) 2024-12-11T20:35:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T20:35:52,509 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/0e10cfdbeead4c4ba9660df4943720d0 2024-12-11T20:35:52,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/953eaa31117f433f8e256e23939f5f67 is 50, key is test_row_0/C:col10/1733949350237/Put/seqid=0 2024-12-11T20:35:52,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742183_1359 (size=12001) 2024-12-11T20:35:52,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949412566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:52,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949412566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949412567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949412567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:52,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949412567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:52,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:52,928 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/953eaa31117f433f8e256e23939f5f67 2024-12-11T20:35:52,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/512d51a2a7154ee49032c08fc4eeea24 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/512d51a2a7154ee49032c08fc4eeea24 2024-12-11T20:35:52,938 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/512d51a2a7154ee49032c08fc4eeea24, entries=150, sequenceid=49, filesize=30.2 K 2024-12-11T20:35:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/0e10cfdbeead4c4ba9660df4943720d0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/0e10cfdbeead4c4ba9660df4943720d0 2024-12-11T20:35:52,945 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/0e10cfdbeead4c4ba9660df4943720d0, entries=150, sequenceid=49, filesize=11.7 K 2024-12-11T20:35:52,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/953eaa31117f433f8e256e23939f5f67 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/953eaa31117f433f8e256e23939f5f67 2024-12-11T20:35:52,950 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/953eaa31117f433f8e256e23939f5f67, entries=150, sequenceid=49, filesize=11.7 K 2024-12-11T20:35:52,951 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 4451ee260f37d1215790a309e039ba65 in 1716ms, sequenceid=49, compaction requested=false 2024-12-11T20:35:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:52,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-11T20:35:52,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-11T20:35:52,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-11T20:35:52,954 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7910 sec 2024-12-11T20:35:52,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.8020 sec 2024-12-11T20:35:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:53,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T20:35:53,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:35:53,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:53,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:35:53,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:53,581 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:35:53,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:53,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211da46933363794c288af29612b03332c1_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949353580/Put/seqid=0 2024-12-11T20:35:53,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742184_1360 (size=12154) 2024-12-11T20:35:53,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949413611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949413611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949413612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949413613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949413619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949413727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949413727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949413727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949413727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949413727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949413939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949413947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,952 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949413948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949413948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:53,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949413948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:53,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:53,992 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:53,996 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211da46933363794c288af29612b03332c1_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211da46933363794c288af29612b03332c1_4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:53,997 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/6886523d0cec4d69b86bdb12dffc02ab, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:53,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/6886523d0cec4d69b86bdb12dffc02ab is 175, key is test_row_0/A:col10/1733949353580/Put/seqid=0 2024-12-11T20:35:54,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742185_1361 (size=30955) 2024-12-11T20:35:54,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:54,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949414257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949414257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:54,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949414257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949414257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:54,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949414267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-11T20:35:54,327 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:54,327 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-11T20:35:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T20:35:54,329 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:54,330 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:54,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:54,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/6886523d0cec4d69b86bdb12dffc02ab 2024-12-11T20:35:54,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T20:35:54,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/fe9fa87b5d144c74aa09560866e7e427 is 50, key is test_row_0/B:col10/1733949353580/Put/seqid=0 2024-12-11T20:35:54,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742186_1362 (size=12001) 2024-12-11T20:35:54,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T20:35:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:54,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:54,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:54,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T20:35:54,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:54,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:54,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:54,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T20:35:54,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949414767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:54,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949414777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:54,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949414777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:54,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949414777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:54,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T20:35:54,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:54,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:54,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:54,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949414786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:54,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:54,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:54,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/fe9fa87b5d144c74aa09560866e7e427 2024-12-11T20:35:54,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/40eb8d9212464a15892e1be918f06d53 is 50, key is test_row_0/C:col10/1733949353580/Put/seqid=0 2024-12-11T20:35:54,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742187_1363 (size=12001) 2024-12-11T20:35:54,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T20:35:54,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:54,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:54,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:54,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T20:35:55,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T20:35:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:55,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:55,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:55,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T20:35:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:55,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:55,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:55,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/40eb8d9212464a15892e1be918f06d53 2024-12-11T20:35:55,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/6886523d0cec4d69b86bdb12dffc02ab as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/6886523d0cec4d69b86bdb12dffc02ab 2024-12-11T20:35:55,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/6886523d0cec4d69b86bdb12dffc02ab, entries=150, sequenceid=61, filesize=30.2 K 2024-12-11T20:35:55,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/fe9fa87b5d144c74aa09560866e7e427 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fe9fa87b5d144c74aa09560866e7e427 2024-12-11T20:35:55,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fe9fa87b5d144c74aa09560866e7e427, entries=150, sequenceid=61, filesize=11.7 K 2024-12-11T20:35:55,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/40eb8d9212464a15892e1be918f06d53 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/40eb8d9212464a15892e1be918f06d53 2024-12-11T20:35:55,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/40eb8d9212464a15892e1be918f06d53, entries=150, sequenceid=61, filesize=11.7 K 2024-12-11T20:35:55,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4451ee260f37d1215790a309e039ba65 in 1722ms, sequenceid=61, compaction requested=true 2024-12-11T20:35:55,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:55,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:55,304 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:55,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:55,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:55,304 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:55,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:55,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:55,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:55,304 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:55,305 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/A is initiating minor compaction (all files) 2024-12-11T20:35:55,305 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/A in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:55,305 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/128d6c41763b4459adae166b279e0f7f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/512d51a2a7154ee49032c08fc4eeea24, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/6886523d0cec4d69b86bdb12dffc02ab] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=90.7 K 2024-12-11T20:35:55,305 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:55,305 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/128d6c41763b4459adae166b279e0f7f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/512d51a2a7154ee49032c08fc4eeea24, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/6886523d0cec4d69b86bdb12dffc02ab] 2024-12-11T20:35:55,305 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:55,305 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/B is initiating minor compaction (all files) 2024-12-11T20:35:55,305 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/B in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:55,305 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e2b1b09406c646eb8c38f9848a0924f7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/0e10cfdbeead4c4ba9660df4943720d0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fe9fa87b5d144c74aa09560866e7e427] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=35.2 K 2024-12-11T20:35:55,305 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting e2b1b09406c646eb8c38f9848a0924f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733949350158 2024-12-11T20:35:55,306 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e10cfdbeead4c4ba9660df4943720d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733949350237 2024-12-11T20:35:55,306 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 128d6c41763b4459adae166b279e0f7f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733949350158 2024-12-11T20:35:55,306 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9fa87b5d144c74aa09560866e7e427, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733949351402 2024-12-11T20:35:55,306 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 512d51a2a7154ee49032c08fc4eeea24, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733949350237 2024-12-11T20:35:55,307 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6886523d0cec4d69b86bdb12dffc02ab, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733949351402 2024-12-11T20:35:55,319 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:55,320 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#B#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:55,320 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/99265d1b05ee4764b761d5cada0c9de5 is 50, key is test_row_0/B:col10/1733949353580/Put/seqid=0 2024-12-11T20:35:55,327 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211d02530da9e50498598545cbbd220a7c5_4451ee260f37d1215790a309e039ba65 store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:55,329 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211d02530da9e50498598545cbbd220a7c5_4451ee260f37d1215790a309e039ba65, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:55,329 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d02530da9e50498598545cbbd220a7c5_4451ee260f37d1215790a309e039ba65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:55,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742188_1364 (size=12104) 2024-12-11T20:35:55,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742189_1365 (size=4469) 2024-12-11T20:35:55,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-11T20:35:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:55,407 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T20:35:55,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:35:55,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:55,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:35:55,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:55,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:35:55,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:55,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412114085cfd9bde14bdb966ba96359cf1c8c_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949353611/Put/seqid=0 2024-12-11T20:35:55,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742190_1366 (size=12154) 2024-12-11T20:35:55,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:55,446 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412114085cfd9bde14bdb966ba96359cf1c8c_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114085cfd9bde14bdb966ba96359cf1c8c_4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:55,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/d472ac5a154a46989aca60e4d3c5c5e2, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:55,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/d472ac5a154a46989aca60e4d3c5c5e2 is 175, key is test_row_0/A:col10/1733949353611/Put/seqid=0 2024-12-11T20:35:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T20:35:55,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742191_1367 (size=30955) 2024-12-11T20:35:55,745 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/99265d1b05ee4764b761d5cada0c9de5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/99265d1b05ee4764b761d5cada0c9de5 2024-12-11T20:35:55,746 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#A#compaction#312 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:55,747 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/b75310c2aef14eacb5d1cae26a59437e is 175, key is test_row_0/A:col10/1733949353580/Put/seqid=0 2024-12-11T20:35:55,750 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/B of 4451ee260f37d1215790a309e039ba65 into 99265d1b05ee4764b761d5cada0c9de5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:55,750 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:55,750 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/B, priority=13, startTime=1733949355304; duration=0sec 2024-12-11T20:35:55,750 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:55,750 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:B 2024-12-11T20:35:55,750 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:55,752 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:55,752 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/C is initiating minor compaction (all files) 2024-12-11T20:35:55,752 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/C in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:55,753 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6f3c52fc7beb4a7b8890f3a770e39e1b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/953eaa31117f433f8e256e23939f5f67, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/40eb8d9212464a15892e1be918f06d53] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=35.2 K 2024-12-11T20:35:55,753 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f3c52fc7beb4a7b8890f3a770e39e1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733949350158 2024-12-11T20:35:55,753 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 953eaa31117f433f8e256e23939f5f67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733949350237 2024-12-11T20:35:55,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742192_1368 (size=31058) 2024-12-11T20:35:55,754 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 40eb8d9212464a15892e1be918f06d53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733949351402 2024-12-11T20:35:55,761 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/b75310c2aef14eacb5d1cae26a59437e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b75310c2aef14eacb5d1cae26a59437e 2024-12-11T20:35:55,763 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#C#compaction#314 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:55,763 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/d0639cabd1974883b0e7e5ce98fc5773 is 50, key is test_row_0/C:col10/1733949353580/Put/seqid=0 2024-12-11T20:35:55,767 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/A of 4451ee260f37d1215790a309e039ba65 into b75310c2aef14eacb5d1cae26a59437e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:55,767 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:55,767 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/A, priority=13, startTime=1733949355303; duration=0sec 2024-12-11T20:35:55,767 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:55,767 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:A 2024-12-11T20:35:55,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742193_1369 (size=12104) 2024-12-11T20:35:55,774 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/d0639cabd1974883b0e7e5ce98fc5773 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/d0639cabd1974883b0e7e5ce98fc5773 2024-12-11T20:35:55,779 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/C of 4451ee260f37d1215790a309e039ba65 into d0639cabd1974883b0e7e5ce98fc5773(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:55,779 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:55,779 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/C, priority=13, startTime=1733949355304; duration=0sec 2024-12-11T20:35:55,779 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:55,780 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:C 2024-12-11T20:35:55,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:55,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:55,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949415807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:55,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949415807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:55,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:55,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949415808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:55,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:55,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949415811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:55,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,817 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:55,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:55,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949415812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:55,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:55,878 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/d472ac5a154a46989aca60e4d3c5c5e2 2024-12-11T20:35:55,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/e27fc3c4847c4a01bcaeef5fddd1999c is 50, key is test_row_0/B:col10/1733949353611/Put/seqid=0 2024-12-11T20:35:55,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742194_1370 (size=12001) 2024-12-11T20:35:55,891 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/e27fc3c4847c4a01bcaeef5fddd1999c 2024-12-11T20:35:55,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/7d844acf9d214531bbd4e325a572f82a is 50, key is test_row_0/C:col10/1733949353611/Put/seqid=0 2024-12-11T20:35:55,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742195_1371 (size=12001) 2024-12-11T20:35:55,913 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/7d844acf9d214531bbd4e325a572f82a 2024-12-11T20:35:55,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/d472ac5a154a46989aca60e4d3c5c5e2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/d472ac5a154a46989aca60e4d3c5c5e2 2024-12-11T20:35:55,923 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/d472ac5a154a46989aca60e4d3c5c5e2, entries=150, sequenceid=81, filesize=30.2 K 2024-12-11T20:35:55,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/e27fc3c4847c4a01bcaeef5fddd1999c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e27fc3c4847c4a01bcaeef5fddd1999c 2024-12-11T20:35:55,928 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e27fc3c4847c4a01bcaeef5fddd1999c, entries=150, sequenceid=81, filesize=11.7 K 2024-12-11T20:35:55,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949415927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949415927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949415927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949415927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:55,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949415927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:55,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:55,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:55,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/7d844acf9d214531bbd4e325a572f82a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/7d844acf9d214531bbd4e325a572f82a 2024-12-11T20:35:55,935 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/7d844acf9d214531bbd4e325a572f82a, entries=150, sequenceid=81, filesize=11.7 K 2024-12-11T20:35:55,936 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 4451ee260f37d1215790a309e039ba65 in 529ms, sequenceid=81, compaction requested=false 2024-12-11T20:35:55,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:55,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:55,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-11T20:35:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-11T20:35:55,938 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-11T20:35:55,938 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6070 sec 2024-12-11T20:35:55,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.6110 sec 2024-12-11T20:35:56,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:56,139 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T20:35:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:35:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:35:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:35:56,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:56,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211cad44166eeba459a9dcc91be69eae9dc_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949355805/Put/seqid=0 2024-12-11T20:35:56,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742196_1372 (size=14594) 2024-12-11T20:35:56,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949416161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949416161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949416163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949416164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949416165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949416277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949416277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949416277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949416277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949416277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-11T20:35:56,487 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:56,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-11T20:35:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T20:35:56,489 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:56,489 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:56,489 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:56,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949416487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949416487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949416487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949416487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949416487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,553 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:56,556 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211cad44166eeba459a9dcc91be69eae9dc_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211cad44166eeba459a9dcc91be69eae9dc_4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:56,557 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/556d66f5ef7241dd96c17f2e10d59eb2, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:56,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/556d66f5ef7241dd96c17f2e10d59eb2 is 175, key is test_row_0/A:col10/1733949355805/Put/seqid=0 2024-12-11T20:35:56,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742197_1373 (size=39549) 2024-12-11T20:35:56,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T20:35:56,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-11T20:35:56,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:56,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:56,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:56,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:56,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:56,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-11T20:35:56,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:56,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:56,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:56,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:56,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:56,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949416797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949416797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949416797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,802 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949416797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949416797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T20:35:56,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-11T20:35:56,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:56,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:56,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:56,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:56,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:56,963 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/556d66f5ef7241dd96c17f2e10d59eb2 2024-12-11T20:35:56,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/f603557fe4764f1981fd2c6ac6802b5b is 50, key is test_row_0/B:col10/1733949355805/Put/seqid=0 2024-12-11T20:35:56,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742198_1374 (size=12001) 2024-12-11T20:35:57,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-11T20:35:57,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:57,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T20:35:57,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-11T20:35:57,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:57,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949417307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:57,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949417307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:57,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:57,310 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949417307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:57,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949417307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:57,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:57,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949417317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:57,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:57,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/f603557fe4764f1981fd2c6ac6802b5b 2024-12-11T20:35:57,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/6da6f9558f124e7490e067c3e6a171a5 is 50, key is test_row_0/C:col10/1733949355805/Put/seqid=0 2024-12-11T20:35:57,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742199_1375 (size=12001) 2024-12-11T20:35:57,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-11T20:35:57,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-11T20:35:57,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:57,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T20:35:57,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-11T20:35:57,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:57,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:57,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/6da6f9558f124e7490e067c3e6a171a5 2024-12-11T20:35:57,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/556d66f5ef7241dd96c17f2e10d59eb2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/556d66f5ef7241dd96c17f2e10d59eb2 2024-12-11T20:35:57,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/556d66f5ef7241dd96c17f2e10d59eb2, entries=200, sequenceid=97, filesize=38.6 K 2024-12-11T20:35:57,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/f603557fe4764f1981fd2c6ac6802b5b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/f603557fe4764f1981fd2c6ac6802b5b 2024-12-11T20:35:57,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/f603557fe4764f1981fd2c6ac6802b5b, entries=150, sequenceid=97, filesize=11.7 K 2024-12-11T20:35:57,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/6da6f9558f124e7490e067c3e6a171a5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6da6f9558f124e7490e067c3e6a171a5 2024-12-11T20:35:57,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6da6f9558f124e7490e067c3e6a171a5, entries=150, sequenceid=97, filesize=11.7 K 2024-12-11T20:35:57,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 4451ee260f37d1215790a309e039ba65 in 1670ms, sequenceid=97, compaction requested=true 2024-12-11T20:35:57,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:57,810 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:35:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:35:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:35:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T20:35:57,810 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:57,810 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:57,811 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/A is initiating minor compaction (all files) 2024-12-11T20:35:57,811 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/A in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,811 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b75310c2aef14eacb5d1cae26a59437e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/d472ac5a154a46989aca60e4d3c5c5e2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/556d66f5ef7241dd96c17f2e10d59eb2] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=99.2 K 2024-12-11T20:35:57,811 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,811 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b75310c2aef14eacb5d1cae26a59437e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/d472ac5a154a46989aca60e4d3c5c5e2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/556d66f5ef7241dd96c17f2e10d59eb2] 2024-12-11T20:35:57,811 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:57,811 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b75310c2aef14eacb5d1cae26a59437e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733949351402 2024-12-11T20:35:57,811 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/B is initiating minor compaction (all files) 2024-12-11T20:35:57,811 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/B in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,811 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/99265d1b05ee4764b761d5cada0c9de5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e27fc3c4847c4a01bcaeef5fddd1999c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/f603557fe4764f1981fd2c6ac6802b5b] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=35.3 K 2024-12-11T20:35:57,811 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting d472ac5a154a46989aca60e4d3c5c5e2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733949353610 2024-12-11T20:35:57,812 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 99265d1b05ee4764b761d5cada0c9de5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733949351402 2024-12-11T20:35:57,812 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 556d66f5ef7241dd96c17f2e10d59eb2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733949355805 2024-12-11T20:35:57,812 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting e27fc3c4847c4a01bcaeef5fddd1999c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733949353610 2024-12-11T20:35:57,813 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f603557fe4764f1981fd2c6ac6802b5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733949355805 2024-12-11T20:35:57,818 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:57,819 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#B#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:57,819 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/ac4cc8ba6bac43a895575187a26b30df is 50, key is test_row_0/B:col10/1733949355805/Put/seqid=0 2024-12-11T20:35:57,823 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024121199bbd3a06bd94d639196b893e534fc3e_4451ee260f37d1215790a309e039ba65 store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:57,825 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024121199bbd3a06bd94d639196b893e534fc3e_4451ee260f37d1215790a309e039ba65, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:57,825 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121199bbd3a06bd94d639196b893e534fc3e_4451ee260f37d1215790a309e039ba65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:57,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742200_1376 (size=12207) 2024-12-11T20:35:57,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742201_1377 (size=4469) 2024-12-11T20:35:57,837 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#A#compaction#321 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:57,838 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/ac4cc8ba6bac43a895575187a26b30df as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/ac4cc8ba6bac43a895575187a26b30df 2024-12-11T20:35:57,838 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/5ecd12e6e8634d6fa18fa00eb737607c is 175, key is test_row_0/A:col10/1733949355805/Put/seqid=0 2024-12-11T20:35:57,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742202_1378 (size=31161) 2024-12-11T20:35:57,844 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/B of 4451ee260f37d1215790a309e039ba65 into ac4cc8ba6bac43a895575187a26b30df(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:57,844 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:57,844 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/B, priority=13, startTime=1733949357810; duration=0sec 2024-12-11T20:35:57,845 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:35:57,845 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:B 2024-12-11T20:35:57,845 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:35:57,846 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:35:57,846 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/C is initiating minor compaction (all files) 2024-12-11T20:35:57,846 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/C in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,846 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/d0639cabd1974883b0e7e5ce98fc5773, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/7d844acf9d214531bbd4e325a572f82a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6da6f9558f124e7490e067c3e6a171a5] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=35.3 K 2024-12-11T20:35:57,847 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d0639cabd1974883b0e7e5ce98fc5773, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1733949351402 2024-12-11T20:35:57,847 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d844acf9d214531bbd4e325a572f82a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733949353610 2024-12-11T20:35:57,847 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da6f9558f124e7490e067c3e6a171a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733949355805 2024-12-11T20:35:57,857 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#C#compaction#322 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:35:57,857 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/f306523e15394f87afa842d3bccf8408 is 50, key is test_row_0/C:col10/1733949355805/Put/seqid=0 2024-12-11T20:35:57,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742203_1379 (size=12207) 2024-12-11T20:35:57,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-11T20:35:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:57,863 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-11T20:35:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:35:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:35:57,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:57,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:35:57,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:57,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115b5d7103804a43789e79853cba9e18a1_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949356160/Put/seqid=0 2024-12-11T20:35:57,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742204_1380 (size=12154) 2024-12-11T20:35:57,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:57,876 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115b5d7103804a43789e79853cba9e18a1_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115b5d7103804a43789e79853cba9e18a1_4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:57,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/261f6aa66691495190d0be2267396774, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:57,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/261f6aa66691495190d0be2267396774 is 175, key is test_row_0/A:col10/1733949356160/Put/seqid=0 2024-12-11T20:35:57,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742205_1381 (size=30955) 2024-12-11T20:35:57,885 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/261f6aa66691495190d0be2267396774 2024-12-11T20:35:57,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/ca65fa65d509440788592bc930cfc670 is 50, key is test_row_0/B:col10/1733949356160/Put/seqid=0 2024-12-11T20:35:57,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742206_1382 (size=12001) 2024-12-11T20:35:57,900 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/ca65fa65d509440788592bc930cfc670 2024-12-11T20:35:57,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/e6c79856a302439b810e4a30a2eacc26 is 50, key is test_row_0/C:col10/1733949356160/Put/seqid=0 2024-12-11T20:35:57,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742207_1383 (size=12001) 2024-12-11T20:35:58,248 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/5ecd12e6e8634d6fa18fa00eb737607c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/5ecd12e6e8634d6fa18fa00eb737607c 2024-12-11T20:35:58,252 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/A of 4451ee260f37d1215790a309e039ba65 into 5ecd12e6e8634d6fa18fa00eb737607c(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:58,252 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:58,252 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/A, priority=13, startTime=1733949357810; duration=0sec 2024-12-11T20:35:58,253 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:58,253 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:A 2024-12-11T20:35:58,266 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/f306523e15394f87afa842d3bccf8408 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/f306523e15394f87afa842d3bccf8408 2024-12-11T20:35:58,272 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/C of 4451ee260f37d1215790a309e039ba65 into f306523e15394f87afa842d3bccf8408(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:35:58,272 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:58,272 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/C, priority=13, startTime=1733949357810; duration=0sec 2024-12-11T20:35:58,272 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:35:58,272 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:C 2024-12-11T20:35:58,312 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/e6c79856a302439b810e4a30a2eacc26 2024-12-11T20:35:58,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/261f6aa66691495190d0be2267396774 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/261f6aa66691495190d0be2267396774 2024-12-11T20:35:58,319 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/261f6aa66691495190d0be2267396774, entries=150, sequenceid=117, filesize=30.2 K 2024-12-11T20:35:58,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/ca65fa65d509440788592bc930cfc670 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/ca65fa65d509440788592bc930cfc670 2024-12-11T20:35:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:58,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,323 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/ca65fa65d509440788592bc930cfc670, entries=150, sequenceid=117, filesize=11.7 K 2024-12-11T20:35:58,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/e6c79856a302439b810e4a30a2eacc26 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/e6c79856a302439b810e4a30a2eacc26 2024-12-11T20:35:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,328 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/e6c79856a302439b810e4a30a2eacc26, entries=150, sequenceid=117, filesize=11.7 K 2024-12-11T20:35:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,329 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=6.71 KB/6870 for 4451ee260f37d1215790a309e039ba65 in 465ms, sequenceid=117, compaction requested=false 2024-12-11T20:35:58,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:35:58,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:58,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-11T20:35:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-11T20:35:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-11T20:35:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8410 sec 2024-12-11T20:35:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.8460 sec 2024-12-11T20:35:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:58,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:35:58,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:35:58,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:58,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:35:58,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:58,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:35:58,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:35:58,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412111098638f2c9a46139fe57a41088198f6_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949358405/Put/seqid=0 2024-12-11T20:35:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742208_1384 (size=21968) 2024-12-11T20:35:58,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733949358488, max timestamp=1733949358499Num uniques 859; 2024-12-11T20:35:58,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733949358489, max timestamp=1733949358499Num uniques 859; 2024-12-11T20:35:58,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733949358489, max timestamp=1733949358499Num uniques 859; 2024-12-11T20:35:58,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T20:35:58,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T20:35:58,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-11T20:35:58,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949418522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949418523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949418524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949418534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949418534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949418636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949418636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-11T20:35:58,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:35:58,647 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:35:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-11T20:35:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T20:35:58,648 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:35:58,649 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:35:58,649 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:35:58,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949418647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949418647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949418647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T20:35:58,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T20:35:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:58,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:58,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:58,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:58,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949418847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949418847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949418857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949418857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949418857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:58,884 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:35:58,887 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412111098638f2c9a46139fe57a41088198f6_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412111098638f2c9a46139fe57a41088198f6_4451ee260f37d1215790a309e039ba65 2024-12-11T20:35:58,887 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/48e6aa56e35244548b9a08a7621cd89b, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:35:58,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/48e6aa56e35244548b9a08a7621cd89b is 175, key is test_row_0/A:col10/1733949358405/Put/seqid=0 2024-12-11T20:35:58,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742209_1385 (size=65373) 2024-12-11T20:35:58,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T20:35:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:58,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:58,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:58,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T20:35:59,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T20:35:59,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:59,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949419157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:59,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949419157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:59,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949419167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:59,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949419167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:59,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:59,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949419176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:59,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T20:35:59,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:59,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T20:35:59,300 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/48e6aa56e35244548b9a08a7621cd89b 2024-12-11T20:35:59,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/2aba3cd9ae6a4e3b853ab2e0c078235b is 50, key is test_row_0/B:col10/1733949358405/Put/seqid=0 2024-12-11T20:35:59,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742210_1386 (size=12051) 2024-12-11T20:35:59,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T20:35:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T20:35:59,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:59,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:59,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949419677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:59,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:59,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949419677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:59,682 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:59,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949419687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:59,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:59,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,695 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949419688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:59,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:35:59,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949419697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:35:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:35:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:35:59,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T20:35:59,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:59,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/2aba3cd9ae6a4e3b853ab2e0c078235b 2024-12-11T20:35:59,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/363cd288e2f34797aa61465057a93c3e is 50, key is test_row_0/C:col10/1733949358405/Put/seqid=0 2024-12-11T20:35:59,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742211_1387 (size=12051) 2024-12-11T20:35:59,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T20:35:59,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T20:35:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:35:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:35:59,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:35:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:00,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T20:36:00,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:00,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:00,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:00,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:00,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:00,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/363cd288e2f34797aa61465057a93c3e 2024-12-11T20:36:00,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/48e6aa56e35244548b9a08a7621cd89b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/48e6aa56e35244548b9a08a7621cd89b 2024-12-11T20:36:00,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/48e6aa56e35244548b9a08a7621cd89b, entries=350, sequenceid=129, filesize=63.8 K 2024-12-11T20:36:00,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/2aba3cd9ae6a4e3b853ab2e0c078235b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2aba3cd9ae6a4e3b853ab2e0c078235b 2024-12-11T20:36:00,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2aba3cd9ae6a4e3b853ab2e0c078235b, entries=150, sequenceid=129, filesize=11.8 K 2024-12-11T20:36:00,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/363cd288e2f34797aa61465057a93c3e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/363cd288e2f34797aa61465057a93c3e 2024-12-11T20:36:00,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/363cd288e2f34797aa61465057a93c3e, entries=150, sequenceid=129, filesize=11.8 K 2024-12-11T20:36:00,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for 4451ee260f37d1215790a309e039ba65 in 1711ms, sequenceid=129, compaction requested=true 2024-12-11T20:36:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:00,155 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:00,155 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:00,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:00,156 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127489 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:00,156 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:00,156 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/A is initiating minor compaction (all files) 2024-12-11T20:36:00,156 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/B is initiating minor compaction (all files) 2024-12-11T20:36:00,156 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/B in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:00,156 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/A in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:00,156 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/ac4cc8ba6bac43a895575187a26b30df, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/ca65fa65d509440788592bc930cfc670, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2aba3cd9ae6a4e3b853ab2e0c078235b] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=35.4 K 2024-12-11T20:36:00,156 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/5ecd12e6e8634d6fa18fa00eb737607c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/261f6aa66691495190d0be2267396774, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/48e6aa56e35244548b9a08a7621cd89b] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=124.5 K 2024-12-11T20:36:00,156 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:00,156 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/5ecd12e6e8634d6fa18fa00eb737607c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/261f6aa66691495190d0be2267396774, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/48e6aa56e35244548b9a08a7621cd89b] 2024-12-11T20:36:00,156 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting ac4cc8ba6bac43a895575187a26b30df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733949355805 2024-12-11T20:36:00,156 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting ca65fa65d509440788592bc930cfc670, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733949356160 2024-12-11T20:36:00,156 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ecd12e6e8634d6fa18fa00eb737607c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733949355805 2024-12-11T20:36:00,157 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aba3cd9ae6a4e3b853ab2e0c078235b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733949358404 2024-12-11T20:36:00,157 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 261f6aa66691495190d0be2267396774, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733949356160 2024-12-11T20:36:00,157 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48e6aa56e35244548b9a08a7621cd89b, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733949358322 2024-12-11T20:36:00,162 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#B#compaction#329 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:00,162 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/b7d5f306d9f54ed19b9db3b8208def93 is 50, key is test_row_0/B:col10/1733949358405/Put/seqid=0 2024-12-11T20:36:00,164 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:00,166 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412116877059b890e4ab898ea5ca172db6e42_4451ee260f37d1215790a309e039ba65 store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:00,167 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412116877059b890e4ab898ea5ca172db6e42_4451ee260f37d1215790a309e039ba65, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:00,167 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412116877059b890e4ab898ea5ca172db6e42_4451ee260f37d1215790a309e039ba65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:00,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742212_1388 (size=12359) 2024-12-11T20:36:00,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742213_1389 (size=4469) 2024-12-11T20:36:00,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-11T20:36:00,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:00,183 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=136.42 KB heapSize=385.38 KB 2024-12-11T20:36:00,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:36:00,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:00,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:36:00,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:00,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:36:00,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:00,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211406c3b2ccc334f30932a9ca6b6fced9d_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949358532/Put/seqid=0 2024-12-11T20:36:00,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742214_1390 (size=12304) 2024-12-11T20:36:00,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,207 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211406c3b2ccc334f30932a9ca6b6fced9d_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211406c3b2ccc334f30932a9ca6b6fced9d_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:00,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/b51498c9596142c7a34a879a365c4066, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:00,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/b51498c9596142c7a34a879a365c4066 is 175, key is test_row_0/A:col10/1733949358532/Put/seqid=0 2024-12-11T20:36:00,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742215_1391 (size=31105) 2024-12-11T20:36:00,211 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/b51498c9596142c7a34a879a365c4066 2024-12-11T20:36:00,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/1e86de5f9b154a119d7e5c46cefbc2a0 is 50, key is test_row_0/B:col10/1733949358532/Put/seqid=0 2024-12-11T20:36:00,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742216_1392 (size=12151) 2024-12-11T20:36:00,232 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/1e86de5f9b154a119d7e5c46cefbc2a0 2024-12-11T20:36:00,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/4e9229ecf4e24c6f914eede897a66f6b is 50, key is test_row_0/C:col10/1733949358532/Put/seqid=0 2024-12-11T20:36:00,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742217_1393 (size=12151) 2024-12-11T20:36:00,574 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/b7d5f306d9f54ed19b9db3b8208def93 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/b7d5f306d9f54ed19b9db3b8208def93 2024-12-11T20:36:00,578 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/B of 4451ee260f37d1215790a309e039ba65 into b7d5f306d9f54ed19b9db3b8208def93(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:00,579 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:00,579 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/B, priority=13, startTime=1733949360155; duration=0sec 2024-12-11T20:36:00,579 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:00,579 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:B 2024-12-11T20:36:00,579 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:00,583 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:00,583 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/C is initiating minor compaction (all files) 2024-12-11T20:36:00,583 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/C in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:00,583 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/f306523e15394f87afa842d3bccf8408, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/e6c79856a302439b810e4a30a2eacc26, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/363cd288e2f34797aa61465057a93c3e] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=35.4 K 2024-12-11T20:36:00,583 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f306523e15394f87afa842d3bccf8408, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733949355805 2024-12-11T20:36:00,583 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:00,584 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/7bcddfaead6f44bdb09a870eb83b291a is 175, key is test_row_0/A:col10/1733949358405/Put/seqid=0 2024-12-11T20:36:00,585 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c79856a302439b810e4a30a2eacc26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733949356160 2024-12-11T20:36:00,585 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 363cd288e2f34797aa61465057a93c3e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733949358404 2024-12-11T20:36:00,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742218_1394 (size=31313) 2024-12-11T20:36:00,593 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/7bcddfaead6f44bdb09a870eb83b291a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/7bcddfaead6f44bdb09a870eb83b291a 2024-12-11T20:36:00,600 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#C#compaction#334 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:00,601 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/93af30d6743f414b8e5928ca5fb6f81a is 50, key is test_row_0/C:col10/1733949358405/Put/seqid=0 2024-12-11T20:36:00,606 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/A of 4451ee260f37d1215790a309e039ba65 into 7bcddfaead6f44bdb09a870eb83b291a(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:00,606 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:00,606 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/A, priority=13, startTime=1733949360155; duration=0sec 2024-12-11T20:36:00,606 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:00,606 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:A 2024-12-11T20:36:00,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742219_1395 (size=12359) 2024-12-11T20:36:00,625 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/93af30d6743f414b8e5928ca5fb6f81a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/93af30d6743f414b8e5928ca5fb6f81a 2024-12-11T20:36:00,633 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/C of 4451ee260f37d1215790a309e039ba65 into 93af30d6743f414b8e5928ca5fb6f81a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:00,633 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:00,634 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/C, priority=13, startTime=1733949360155; duration=0sec 2024-12-11T20:36:00,634 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:00,634 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:C 2024-12-11T20:36:00,643 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/4e9229ecf4e24c6f914eede897a66f6b 2024-12-11T20:36:00,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/b51498c9596142c7a34a879a365c4066 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b51498c9596142c7a34a879a365c4066 2024-12-11T20:36:00,654 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b51498c9596142c7a34a879a365c4066, entries=150, sequenceid=165, filesize=30.4 K 2024-12-11T20:36:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/1e86de5f9b154a119d7e5c46cefbc2a0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/1e86de5f9b154a119d7e5c46cefbc2a0 2024-12-11T20:36:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,658 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/1e86de5f9b154a119d7e5c46cefbc2a0, entries=150, sequenceid=165, filesize=11.9 K 2024-12-11T20:36:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/4e9229ecf4e24c6f914eede897a66f6b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4e9229ecf4e24c6f914eede897a66f6b 2024-12-11T20:36:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,663 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4e9229ecf4e24c6f914eede897a66f6b, entries=150, sequenceid=165, filesize=11.9 K 2024-12-11T20:36:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,664 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~385.33 KB/394576, currentSize=0 B/0 for 4451ee260f37d1215790a309e039ba65 in 481ms, sequenceid=165, compaction requested=false 2024-12-11T20:36:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:00,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:00,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-11T20:36:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-11T20:36:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-11T20:36:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0180 sec 2024-12-11T20:36:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.0220 sec 2024-12-11T20:36:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-11T20:36:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,797 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:00,798 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-11T20:36:00,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T20:36:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:36:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:36:00,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:00,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:36:00,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:00,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:36:00,806 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:00,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:00,807 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:00,808 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:00,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412119a3e9612a1ee48768e07368ce0454869_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949360761/Put/seqid=0 2024-12-11T20:36:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742220_1396 (size=22268) 2024-12-11T20:36:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,827 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412119a3e9612a1ee48768e07368ce0454869_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412119a3e9612a1ee48768e07368ce0454869_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:00,831 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/705dbd1dc56c4f6eb9607d2ce62ca1f0, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:00,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/705dbd1dc56c4f6eb9607d2ce62ca1f0 is 175, key is test_row_0/A:col10/1733949360761/Put/seqid=0 2024-12-11T20:36:00,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742221_1397 (size=65673) 2024-12-11T20:36:00,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:00,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733949360829, max timestamp=1733949360862Num uniques 859; 2024-12-11T20:36:00,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733949360829, max timestamp=1733949360862Num uniques 859; 2024-12-11T20:36:00,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T20:36:00,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-11T20:36:00,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=179, cellSize=8194, totalHeapSize=18856, min timestamp=1733949360829, max timestamp=1733949360862Num uniques 859; 2024-12-11T20:36:00,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8194, suffix heap size=117064,new segment heap size=18856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=179 2024-12-11T20:36:00,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T20:36:00,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:00,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949420898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949420903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:00,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949420938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:00,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949420938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:00,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949420938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:00,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:00,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-11T20:36:00,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:00,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:00,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:00,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:00,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:00,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949421047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949421047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949421057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949421057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949421057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-11T20:36:01,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:01,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T20:36:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,235 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=178, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/705dbd1dc56c4f6eb9607d2ce62ca1f0 2024-12-11T20:36:01,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/fad0994532be43818f6b4c5cb1e04eae is 50, key is test_row_0/B:col10/1733949360761/Put/seqid=0 2024-12-11T20:36:01,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742222_1398 (size=12151) 2024-12-11T20:36:01,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/fad0994532be43818f6b4c5cb1e04eae 2024-12-11T20:36:01,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/6c5da9c6fa844764bfa4fb7195ab4284 is 50, key is test_row_0/C:col10/1733949360761/Put/seqid=0 2024-12-11T20:36:01,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949421257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949421257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742223_1399 (size=12151) 2024-12-11T20:36:01,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-11T20:36:01,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:01,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949421277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949421278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949421278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-11T20:36:01,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T20:36:01,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949421567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949421567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-11T20:36:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:01,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949421586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949421587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:01,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949421587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:01,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/6c5da9c6fa844764bfa4fb7195ab4284 2024-12-11T20:36:01,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/705dbd1dc56c4f6eb9607d2ce62ca1f0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/705dbd1dc56c4f6eb9607d2ce62ca1f0 2024-12-11T20:36:01,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/705dbd1dc56c4f6eb9607d2ce62ca1f0, entries=350, sequenceid=178, filesize=64.1 K 2024-12-11T20:36:01,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/fad0994532be43818f6b4c5cb1e04eae as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fad0994532be43818f6b4c5cb1e04eae 2024-12-11T20:36:01,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fad0994532be43818f6b4c5cb1e04eae, entries=150, sequenceid=178, filesize=11.9 K 2024-12-11T20:36:01,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/6c5da9c6fa844764bfa4fb7195ab4284 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6c5da9c6fa844764bfa4fb7195ab4284 2024-12-11T20:36:01,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6c5da9c6fa844764bfa4fb7195ab4284, entries=150, sequenceid=178, filesize=11.9 K 2024-12-11T20:36:01,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=144.02 KB/147480 for 4451ee260f37d1215790a309e039ba65 in 880ms, sequenceid=178, compaction requested=true 2024-12-11T20:36:01,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:01,685 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:01,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:01,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:01,685 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:01,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:01,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:01,686 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128091 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:01,686 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/A is initiating minor compaction (all files) 2024-12-11T20:36:01,686 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:01,686 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/A in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,686 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/B is initiating minor compaction (all files) 2024-12-11T20:36:01,686 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/B in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,686 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/7bcddfaead6f44bdb09a870eb83b291a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b51498c9596142c7a34a879a365c4066, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/705dbd1dc56c4f6eb9607d2ce62ca1f0] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=125.1 K 2024-12-11T20:36:01,686 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/b7d5f306d9f54ed19b9db3b8208def93, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/1e86de5f9b154a119d7e5c46cefbc2a0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fad0994532be43818f6b4c5cb1e04eae] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=35.8 K 2024-12-11T20:36:01,686 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,686 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/7bcddfaead6f44bdb09a870eb83b291a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b51498c9596142c7a34a879a365c4066, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/705dbd1dc56c4f6eb9607d2ce62ca1f0] 2024-12-11T20:36:01,686 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting b7d5f306d9f54ed19b9db3b8208def93, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733949358404 2024-12-11T20:36:01,687 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bcddfaead6f44bdb09a870eb83b291a, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733949358404 2024-12-11T20:36:01,687 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e86de5f9b154a119d7e5c46cefbc2a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733949358510 2024-12-11T20:36:01,687 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b51498c9596142c7a34a879a365c4066, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733949358510 2024-12-11T20:36:01,687 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 705dbd1dc56c4f6eb9607d2ce62ca1f0, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733949360760 2024-12-11T20:36:01,689 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting fad0994532be43818f6b4c5cb1e04eae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733949360761 2024-12-11T20:36:01,698 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:01,709 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#B#compaction#339 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:01,709 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/e37ae65d190148ef96cc192a4b57629f is 50, key is test_row_0/B:col10/1733949360761/Put/seqid=0 2024-12-11T20:36:01,711 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211e87cb676c94f451480f65f505a0b50de_4451ee260f37d1215790a309e039ba65 store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:01,713 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211e87cb676c94f451480f65f505a0b50de_4451ee260f37d1215790a309e039ba65, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:01,713 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e87cb676c94f451480f65f505a0b50de_4451ee260f37d1215790a309e039ba65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:01,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-11T20:36:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,729 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=144.02 KB heapSize=407.33 KB 2024-12-11T20:36:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:36:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:36:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:36:01,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:01,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742224_1400 (size=12561) 2024-12-11T20:36:01,759 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/e37ae65d190148ef96cc192a4b57629f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e37ae65d190148ef96cc192a4b57629f 2024-12-11T20:36:01,764 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/B of 4451ee260f37d1215790a309e039ba65 into e37ae65d190148ef96cc192a4b57629f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:01,764 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:01,764 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/B, priority=13, startTime=1733949361685; duration=0sec 2024-12-11T20:36:01,764 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:01,764 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:B 2024-12-11T20:36:01,764 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:01,765 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:01,765 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/C is initiating minor compaction (all files) 2024-12-11T20:36:01,765 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/C in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:01,765 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/93af30d6743f414b8e5928ca5fb6f81a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4e9229ecf4e24c6f914eede897a66f6b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6c5da9c6fa844764bfa4fb7195ab4284] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=35.8 K 2024-12-11T20:36:01,766 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 93af30d6743f414b8e5928ca5fb6f81a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733949358404 2024-12-11T20:36:01,766 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e9229ecf4e24c6f914eede897a66f6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733949358510 2024-12-11T20:36:01,766 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c5da9c6fa844764bfa4fb7195ab4284, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733949360761 2024-12-11T20:36:01,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742225_1401 (size=4469) 2024-12-11T20:36:01,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412111ba0cb0627cd414ab150b30e564d334f_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949360897/Put/seqid=0 2024-12-11T20:36:01,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742226_1402 (size=12304) 2024-12-11T20:36:01,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:01,809 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:01,809 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412111ba0cb0627cd414ab150b30e564d334f_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412111ba0cb0627cd414ab150b30e564d334f_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:01,809 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/4c14053c3a734290a534f7edbaa6f7bc is 50, key is test_row_0/C:col10/1733949360761/Put/seqid=0 2024-12-11T20:36:01,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/294c954df49b4d3c938675e14d7a8f15, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:01,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/294c954df49b4d3c938675e14d7a8f15 is 175, key is test_row_0/A:col10/1733949360897/Put/seqid=0 2024-12-11T20:36:01,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742227_1403 (size=31105) 2024-12-11T20:36:01,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742228_1404 (size=12561) 2024-12-11T20:36:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T20:36:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:02,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:02,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949422101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949422102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949422104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949422107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949422107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,169 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#A#compaction#338 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:02,169 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/aeb90e10a95b4ee0b35ba8bf544a2fef is 175, key is test_row_0/A:col10/1733949360761/Put/seqid=0 2024-12-11T20:36:02,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742229_1405 (size=31515) 2024-12-11T20:36:02,228 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/294c954df49b4d3c938675e14d7a8f15 2024-12-11T20:36:02,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/7058db94df0347e6ae90ccac970b7d3d is 50, key is test_row_0/B:col10/1733949360897/Put/seqid=0 2024-12-11T20:36:02,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949422222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949422222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949422222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,238 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/4c14053c3a734290a534f7edbaa6f7bc as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4c14053c3a734290a534f7edbaa6f7bc 2024-12-11T20:36:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,243 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/C of 4451ee260f37d1215790a309e039ba65 into 4c14053c3a734290a534f7edbaa6f7bc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:02,243 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:02,243 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/C, priority=13, startTime=1733949361686; duration=0sec 2024-12-11T20:36:02,243 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:02,243 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:C 2024-12-11T20:36:02,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742230_1406 (size=12151) 2024-12-11T20:36:02,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949422446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949422446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949422446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,452 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,577 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/aeb90e10a95b4ee0b35ba8bf544a2fef as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/aeb90e10a95b4ee0b35ba8bf544a2fef 2024-12-11T20:36:02,581 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/A of 4451ee260f37d1215790a309e039ba65 into aeb90e10a95b4ee0b35ba8bf544a2fef(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:02,582 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:02,582 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/A, priority=13, startTime=1733949361685; duration=0sec 2024-12-11T20:36:02,582 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:02,582 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:A 2024-12-11T20:36:02,643 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/7058db94df0347e6ae90ccac970b7d3d 2024-12-11T20:36:02,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/5cf5766e778b4ba183cf8a722da2146b is 50, key is test_row_0/C:col10/1733949360897/Put/seqid=0 2024-12-11T20:36:02,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742231_1407 (size=12151) 2024-12-11T20:36:02,654 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=45.63 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/5cf5766e778b4ba183cf8a722da2146b 2024-12-11T20:36:02,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/294c954df49b4d3c938675e14d7a8f15 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/294c954df49b4d3c938675e14d7a8f15 2024-12-11T20:36:02,661 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/294c954df49b4d3c938675e14d7a8f15, entries=150, sequenceid=215, filesize=30.4 K 2024-12-11T20:36:02,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/7058db94df0347e6ae90ccac970b7d3d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/7058db94df0347e6ae90ccac970b7d3d 2024-12-11T20:36:02,669 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/7058db94df0347e6ae90ccac970b7d3d, entries=150, sequenceid=215, filesize=11.9 K 2024-12-11T20:36:02,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/5cf5766e778b4ba183cf8a722da2146b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/5cf5766e778b4ba183cf8a722da2146b 2024-12-11T20:36:02,673 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/5cf5766e778b4ba183cf8a722da2146b, entries=150, sequenceid=215, filesize=11.9 K 2024-12-11T20:36:02,674 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~144.02 KB/147480, heapSize ~407.28 KB/417056, currentSize=40.25 KB/41220 for 4451ee260f37d1215790a309e039ba65 in 945ms, sequenceid=215, compaction requested=false 2024-12-11T20:36:02,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:02,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:02,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-11T20:36:02,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-11T20:36:02,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-11T20:36:02,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8690 sec 2024-12-11T20:36:02,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.8810 sec 2024-12-11T20:36:02,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:02,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:36:02,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:36:02,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:02,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:36:02,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:02,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:36:02,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:02,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121128ca2bf5f7f64191ad3ad7f1c4313f66_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949362763/Put/seqid=0 2024-12-11T20:36:02,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742232_1408 (size=17284) 2024-12-11T20:36:02,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949422802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949362780, max timestamp=1733949362807Num uniques 859; 2024-12-11T20:36:02,809 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:02,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949362780, max timestamp=1733949362807Num uniques 859; 2024-12-11T20:36:02,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:02,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949362780, max timestamp=1733949362807Num uniques 859; 2024-12-11T20:36:02,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:02,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949422893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949422894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:02,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949422916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:02,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-11T20:36:02,947 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:02,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:02,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-11T20:36:02,949 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-11T20:36:02,949 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:02,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:03,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:03,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949422997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:03,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:03,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949423007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:03,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-11T20:36:03,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-11T20:36:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949423126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:03,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949423126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:03,132 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in 4451ee260f37d1215790a309e039ba65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:03,133 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_0' in 4451ee260f37d1215790a309e039ba65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:03,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949423137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:03,177 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:03,181 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121128ca2bf5f7f64191ad3ad7f1c4313f66_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121128ca2bf5f7f64191ad3ad7f1c4313f66_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:03,182 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/1f5f019574424597a54e98379a072411, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:03,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/1f5f019574424597a54e98379a072411 is 175, key is test_row_0/A:col10/1733949362763/Put/seqid=0 2024-12-11T20:36:03,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742233_1409 (size=48389) 2024-12-11T20:36:03,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949423216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:03,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949423216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:03,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-11T20:36:03,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:03,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-11T20:36:03,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-11T20:36:03,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:03,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:03,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949423446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,452 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:03,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949423537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:03,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949423537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:03,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:03,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-11T20:36:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-11T20:36:03,586 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=229, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/1f5f019574424597a54e98379a072411 2024-12-11T20:36:03,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/33ce4b46b74848a389d38554179936db is 50, key is test_row_0/B:col10/1733949362763/Put/seqid=0 2024-12-11T20:36:03,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742234_1410 (size=12151) 2024-12-11T20:36:03,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/33ce4b46b74848a389d38554179936db 2024-12-11T20:36:03,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/4ca042ab35924f8494f1a29d5663b47c is 50, key is test_row_0/C:col10/1733949362763/Put/seqid=0 2024-12-11T20:36:03,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742235_1411 (size=12151) 2024-12-11T20:36:03,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/4ca042ab35924f8494f1a29d5663b47c 2024-12-11T20:36:03,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/1f5f019574424597a54e98379a072411 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/1f5f019574424597a54e98379a072411 2024-12-11T20:36:03,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/1f5f019574424597a54e98379a072411, entries=250, sequenceid=229, filesize=47.3 K 2024-12-11T20:36:03,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/33ce4b46b74848a389d38554179936db as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/33ce4b46b74848a389d38554179936db 2024-12-11T20:36:03,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/33ce4b46b74848a389d38554179936db, entries=150, sequenceid=229, filesize=11.9 K 2024-12-11T20:36:03,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/4ca042ab35924f8494f1a29d5663b47c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4ca042ab35924f8494f1a29d5663b47c 2024-12-11T20:36:03,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4ca042ab35924f8494f1a29d5663b47c, entries=150, sequenceid=229, filesize=11.9 K 2024-12-11T20:36:03,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4451ee260f37d1215790a309e039ba65 in 909ms, sequenceid=229, compaction requested=true 2024-12-11T20:36:03,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:03,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:03,673 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:03,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:03,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:03,673 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:03,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:03,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:03,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:03,674 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:03,674 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/A is initiating minor compaction (all files) 2024-12-11T20:36:03,674 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/A in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,674 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:03,674 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/B is initiating minor compaction (all files) 2024-12-11T20:36:03,674 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/aeb90e10a95b4ee0b35ba8bf544a2fef, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/294c954df49b4d3c938675e14d7a8f15, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/1f5f019574424597a54e98379a072411] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=108.4 K 2024-12-11T20:36:03,674 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/B in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,674 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e37ae65d190148ef96cc192a4b57629f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/7058db94df0347e6ae90ccac970b7d3d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/33ce4b46b74848a389d38554179936db] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=36.0 K 2024-12-11T20:36:03,674 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,674 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/aeb90e10a95b4ee0b35ba8bf544a2fef, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/294c954df49b4d3c938675e14d7a8f15, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/1f5f019574424597a54e98379a072411] 2024-12-11T20:36:03,674 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting e37ae65d190148ef96cc192a4b57629f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733949360761 2024-12-11T20:36:03,674 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeb90e10a95b4ee0b35ba8bf544a2fef, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733949360761 2024-12-11T20:36:03,674 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 7058db94df0347e6ae90ccac970b7d3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733949360897 2024-12-11T20:36:03,675 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 33ce4b46b74848a389d38554179936db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733949362097 2024-12-11T20:36:03,675 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 294c954df49b4d3c938675e14d7a8f15, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733949360897 2024-12-11T20:36:03,676 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f5f019574424597a54e98379a072411, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733949362097 2024-12-11T20:36:03,681 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:03,682 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#B#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:03,682 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/218f78eba8c04711990136d9dc7d9e7e is 50, key is test_row_0/B:col10/1733949362763/Put/seqid=0 2024-12-11T20:36:03,684 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211484ea2752d474f69b3dc9af0bb91b290_4451ee260f37d1215790a309e039ba65 store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:03,686 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211484ea2752d474f69b3dc9af0bb91b290_4451ee260f37d1215790a309e039ba65, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:03,686 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211484ea2752d474f69b3dc9af0bb91b290_4451ee260f37d1215790a309e039ba65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:03,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742236_1412 (size=12663) 2024-12-11T20:36:03,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742237_1413 (size=4469) 2024-12-11T20:36:03,713 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#A#compaction#348 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:03,714 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/bf9796b592f442338985120ca8bc624a is 175, key is test_row_0/A:col10/1733949362763/Put/seqid=0 2024-12-11T20:36:03,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-11T20:36:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,715 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-11T20:36:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:36:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:36:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:03,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:36:03,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:03,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742238_1414 (size=31617) 2024-12-11T20:36:03,744 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/bf9796b592f442338985120ca8bc624a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/bf9796b592f442338985120ca8bc624a 2024-12-11T20:36:03,748 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/A of 4451ee260f37d1215790a309e039ba65 into bf9796b592f442338985120ca8bc624a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:03,748 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:03,748 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/A, priority=13, startTime=1733949363672; duration=0sec 2024-12-11T20:36:03,748 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:03,749 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:A 2024-12-11T20:36:03,749 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:03,749 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:03,749 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/C is initiating minor compaction (all files) 2024-12-11T20:36:03,749 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/C in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:03,750 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4c14053c3a734290a534f7edbaa6f7bc, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/5cf5766e778b4ba183cf8a722da2146b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4ca042ab35924f8494f1a29d5663b47c] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=36.0 K 2024-12-11T20:36:03,750 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c14053c3a734290a534f7edbaa6f7bc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733949360761 2024-12-11T20:36:03,750 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cf5766e778b4ba183cf8a722da2146b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1733949360897 2024-12-11T20:36:03,750 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ca042ab35924f8494f1a29d5663b47c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733949362097 2024-12-11T20:36:03,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121118ddef89d3e04db2ad05ad560815a74c_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949362839/Put/seqid=0 2024-12-11T20:36:03,759 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:03,759 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/d86ecb52f9694f3ba8a570f0b59a1125 is 50, key is test_row_0/C:col10/1733949362763/Put/seqid=0 2024-12-11T20:36:03,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742239_1415 (size=12404) 2024-12-11T20:36:03,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:03,765 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121118ddef89d3e04db2ad05ad560815a74c_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121118ddef89d3e04db2ad05ad560815a74c_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/9ee8fa2610604885bc64add7c8415ae7, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:03,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/9ee8fa2610604885bc64add7c8415ae7 is 175, key is test_row_0/A:col10/1733949362839/Put/seqid=0 2024-12-11T20:36:03,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742240_1416 (size=12663) 2024-12-11T20:36:03,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742241_1417 (size=31205) 2024-12-11T20:36:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:03,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:04,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:04,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949424013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:04,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949424046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:04,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949424047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:04,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:04,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-11T20:36:04,105 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/218f78eba8c04711990136d9dc7d9e7e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/218f78eba8c04711990136d9dc7d9e7e 2024-12-11T20:36:04,110 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/B of 4451ee260f37d1215790a309e039ba65 into 218f78eba8c04711990136d9dc7d9e7e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:04,110 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:04,110 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/B, priority=13, startTime=1733949363673; duration=0sec 2024-12-11T20:36:04,110 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:04,110 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:B 2024-12-11T20:36:04,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949424126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:04,179 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/d86ecb52f9694f3ba8a570f0b59a1125 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/d86ecb52f9694f3ba8a570f0b59a1125 2024-12-11T20:36:04,185 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/9ee8fa2610604885bc64add7c8415ae7 2024-12-11T20:36:04,197 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/C of 4451ee260f37d1215790a309e039ba65 into d86ecb52f9694f3ba8a570f0b59a1125(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:04,197 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:04,197 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/C, priority=13, startTime=1733949363673; duration=0sec 2024-12-11T20:36:04,197 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:04,197 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:C 2024-12-11T20:36:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/3ef513c954fe44ec96b8771dd933ad6d is 50, key is test_row_0/B:col10/1733949362839/Put/seqid=0 2024-12-11T20:36:04,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742242_1418 (size=12251) 2024-12-11T20:36:04,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:04,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949424346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:04,636 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/3ef513c954fe44ec96b8771dd933ad6d 2024-12-11T20:36:04,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/3a0f6640913b424bac2c1a1efdb67a35 is 50, key is test_row_0/C:col10/1733949362839/Put/seqid=0 2024-12-11T20:36:04,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742243_1419 (size=12251) 2024-12-11T20:36:04,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949424657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,056 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/3a0f6640913b424bac2c1a1efdb67a35 2024-12-11T20:36:05,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/9ee8fa2610604885bc64add7c8415ae7 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/9ee8fa2610604885bc64add7c8415ae7 2024-12-11T20:36:05,062 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/9ee8fa2610604885bc64add7c8415ae7, entries=150, sequenceid=265, filesize=30.5 K 2024-12-11T20:36:05,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/3ef513c954fe44ec96b8771dd933ad6d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/3ef513c954fe44ec96b8771dd933ad6d 2024-12-11T20:36:05,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,067 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/3ef513c954fe44ec96b8771dd933ad6d, entries=150, sequenceid=265, filesize=12.0 K 2024-12-11T20:36:05,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/3a0f6640913b424bac2c1a1efdb67a35 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/3a0f6640913b424bac2c1a1efdb67a35 2024-12-11T20:36:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,071 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/3a0f6640913b424bac2c1a1efdb67a35, entries=150, sequenceid=265, filesize=12.0 K 2024-12-11T20:36:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,072 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 4451ee260f37d1215790a309e039ba65 in 1357ms, sequenceid=265, compaction requested=false 2024-12-11T20:36:05,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:05,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:05,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-11T20:36:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-11T20:36:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-11T20:36:05,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1240 sec 2024-12-11T20:36:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.1340 sec 2024-12-11T20:36:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-11T20:36:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,097 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-11T20:36:05,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-11T20:36:05,099 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,100 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:05,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T20:36:05,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:36:05,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:05,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:36:05,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:05,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:36:05,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211a0b4908fff1e448da85ce88674717c3e_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_1/A:col10/1733949365104/Put/seqid=0 2024-12-11T20:36:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742244_1420 (size=12454) 2024-12-11T20:36:05,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-11T20:36:05,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949365210, max timestamp=1733949365213Num uniques 859; 2024-12-11T20:36:05,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949365210, max timestamp=1733949365213Num uniques 859; 2024-12-11T20:36:05,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:05,221 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:05,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949365210, max timestamp=1733949365213Num uniques 859; 2024-12-11T20:36:05,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:05,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-11T20:36:05,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:05,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:05,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:05,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949425244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949425257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949425258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949425259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949425260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949425367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949425377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949425377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949425378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,383 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949425378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-11T20:36:05,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:05,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:05,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:05,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-11T20:36:05,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-11T20:36:05,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:05,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:05,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:05,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,561 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:05,564 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211a0b4908fff1e448da85ce88674717c3e_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211a0b4908fff1e448da85ce88674717c3e_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:05,565 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/78eae041fb5641fbbc8e3d63c2331707, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:05,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/78eae041fb5641fbbc8e3d63c2331707 is 175, key is test_row_1/A:col10/1733949365104/Put/seqid=0 2024-12-11T20:36:05,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742245_1421 (size=31251) 2024-12-11T20:36:05,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949425577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949425587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949425587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949425587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949425587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-11T20:36:05,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:05,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:05,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:05,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-11T20:36:05,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-11T20:36:05,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:05,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:05,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:05,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:05,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949425887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949425897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949425906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949425906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949425907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:05,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:05,972 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=280, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/78eae041fb5641fbbc8e3d63c2331707 2024-12-11T20:36:05,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/b517c84e2c724ca1b5d69ee9742290ba is 50, key is test_row_1/B:col10/1733949365104/Put/seqid=0 2024-12-11T20:36:05,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742246_1422 (size=9857) 2024-12-11T20:36:06,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-11T20:36:06,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:06,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:06,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:06,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:06,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:06,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-11T20:36:06,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:06,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:06,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:06,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:06,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:06,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-11T20:36:06,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-11T20:36:06,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:06,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:06,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:06,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:06,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:06,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:06,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/b517c84e2c724ca1b5d69ee9742290ba 2024-12-11T20:36:06,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/bce74aed360b4337ad96c4431d1acf08 is 50, key is test_row_1/C:col10/1733949365104/Put/seqid=0 2024-12-11T20:36:06,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742247_1423 (size=9857) 2024-12-11T20:36:06,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/bce74aed360b4337ad96c4431d1acf08 2024-12-11T20:36:06,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:06,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949426419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:06,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:06,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949426419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:06,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:06,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949426428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:06,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:06,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949426428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:06,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:06,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949426430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:06,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:06,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:06,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/78eae041fb5641fbbc8e3d63c2331707 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/78eae041fb5641fbbc8e3d63c2331707 2024-12-11T20:36:06,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/78eae041fb5641fbbc8e3d63c2331707, entries=150, sequenceid=280, filesize=30.5 K 2024-12-11T20:36:06,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/b517c84e2c724ca1b5d69ee9742290ba as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/b517c84e2c724ca1b5d69ee9742290ba 2024-12-11T20:36:06,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/b517c84e2c724ca1b5d69ee9742290ba, entries=100, sequenceid=280, filesize=9.6 K 2024-12-11T20:36:06,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/bce74aed360b4337ad96c4431d1acf08 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/bce74aed360b4337ad96c4431d1acf08 2024-12-11T20:36:06,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/bce74aed360b4337ad96c4431d1acf08, entries=100, sequenceid=280, filesize=9.6 K 2024-12-11T20:36:06,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 4451ee260f37d1215790a309e039ba65 in 1370ms, sequenceid=280, compaction requested=true 2024-12-11T20:36:06,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:06,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:06,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:06,477 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:06,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:06,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:06,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:06,477 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:06,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-11T20:36:06,480 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:06,480 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/B is initiating minor compaction (all files) 2024-12-11T20:36:06,480 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/B in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:06,480 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/218f78eba8c04711990136d9dc7d9e7e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/3ef513c954fe44ec96b8771dd933ad6d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/b517c84e2c724ca1b5d69ee9742290ba] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=34.0 K 2024-12-11T20:36:06,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-11T20:36:06,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:06,481 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-12-11T20:36:06,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:36:06,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:06,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:36:06,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:06,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:36:06,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:06,482 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:06,482 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/A is initiating minor compaction (all files) 2024-12-11T20:36:06,482 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/A in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:06,482 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/bf9796b592f442338985120ca8bc624a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/9ee8fa2610604885bc64add7c8415ae7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/78eae041fb5641fbbc8e3d63c2331707] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=91.9 K 2024-12-11T20:36:06,482 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:06,483 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/bf9796b592f442338985120ca8bc624a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/9ee8fa2610604885bc64add7c8415ae7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/78eae041fb5641fbbc8e3d63c2331707] 2024-12-11T20:36:06,483 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 218f78eba8c04711990136d9dc7d9e7e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733949362097 2024-12-11T20:36:06,483 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ef513c954fe44ec96b8771dd933ad6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733949362839 2024-12-11T20:36:06,485 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting b517c84e2c724ca1b5d69ee9742290ba, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733949364011 2024-12-11T20:36:06,485 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf9796b592f442338985120ca8bc624a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733949362097 2024-12-11T20:36:06,486 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ee8fa2610604885bc64add7c8415ae7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733949362839 2024-12-11T20:36:06,488 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78eae041fb5641fbbc8e3d63c2331707, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733949363988 2024-12-11T20:36:06,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211b95d8a5e92eb4bd58253f8283a469196_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949365258/Put/seqid=0 2024-12-11T20:36:06,511 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#B#compaction#357 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:06,512 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/63deb2e45b0548c4a602a45c8f8fadf5 is 50, key is test_row_0/B:col10/1733949362839/Put/seqid=0 2024-12-11T20:36:06,527 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:06,542 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412117bf3dbe724284bb6b8064e3581b3d388_4451ee260f37d1215790a309e039ba65 store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:06,543 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412117bf3dbe724284bb6b8064e3581b3d388_4451ee260f37d1215790a309e039ba65, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:06,543 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412117bf3dbe724284bb6b8064e3581b3d388_4451ee260f37d1215790a309e039ba65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:06,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742248_1424 (size=12454) 2024-12-11T20:36:06,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:06,550 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211b95d8a5e92eb4bd58253f8283a469196_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b95d8a5e92eb4bd58253f8283a469196_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/c4bc88f6a052477582ea2f77fbbff802, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:06,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/c4bc88f6a052477582ea2f77fbbff802 is 175, key is test_row_0/A:col10/1733949365258/Put/seqid=0 2024-12-11T20:36:06,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742249_1425 (size=12865) 2024-12-11T20:36:06,567 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/63deb2e45b0548c4a602a45c8f8fadf5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/63deb2e45b0548c4a602a45c8f8fadf5 2024-12-11T20:36:06,573 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/B of 4451ee260f37d1215790a309e039ba65 into 63deb2e45b0548c4a602a45c8f8fadf5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:06,573 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:06,573 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/B, priority=13, startTime=1733949366477; duration=0sec 2024-12-11T20:36:06,573 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:06,573 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:B 2024-12-11T20:36:06,573 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:06,575 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34771 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:06,575 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/C is initiating minor compaction (all files) 2024-12-11T20:36:06,575 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/C in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:06,575 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/d86ecb52f9694f3ba8a570f0b59a1125, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/3a0f6640913b424bac2c1a1efdb67a35, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/bce74aed360b4337ad96c4431d1acf08] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=34.0 K 2024-12-11T20:36:06,575 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d86ecb52f9694f3ba8a570f0b59a1125, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733949362097 2024-12-11T20:36:06,576 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a0f6640913b424bac2c1a1efdb67a35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1733949362839 2024-12-11T20:36:06,576 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting bce74aed360b4337ad96c4431d1acf08, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733949364011 2024-12-11T20:36:06,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742251_1427 (size=31255) 2024-12-11T20:36:06,583 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/c4bc88f6a052477582ea2f77fbbff802 2024-12-11T20:36:06,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742250_1426 (size=4469) 2024-12-11T20:36:06,601 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#C#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:06,601 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/2de41a7ce6444121aad046a46c441990 is 50, key is test_row_0/C:col10/1733949362839/Put/seqid=0 2024-12-11T20:36:06,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/23cfd0320ca245c29c3da2bd7e58aa74 is 50, key is test_row_0/B:col10/1733949365258/Put/seqid=0 2024-12-11T20:36:06,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742252_1428 (size=12865) 2024-12-11T20:36:06,621 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/2de41a7ce6444121aad046a46c441990 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/2de41a7ce6444121aad046a46c441990 2024-12-11T20:36:06,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742253_1429 (size=12301) 2024-12-11T20:36:06,627 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/C of 4451ee260f37d1215790a309e039ba65 into 2de41a7ce6444121aad046a46c441990(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:06,627 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:06,627 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/C, priority=13, startTime=1733949366477; duration=0sec 2024-12-11T20:36:06,627 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:06,627 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:C 2024-12-11T20:36:06,993 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#A#compaction#358 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:06,994 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/ce8a8556e645423cb2aa6ebc5f170db7 is 175, key is test_row_0/A:col10/1733949362839/Put/seqid=0 2024-12-11T20:36:06,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742254_1430 (size=31926) 2024-12-11T20:36:07,023 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/23cfd0320ca245c29c3da2bd7e58aa74 2024-12-11T20:36:07,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/ef3dac9f3828410da378e66cb29c4452 is 50, key is test_row_0/C:col10/1733949365258/Put/seqid=0 2024-12-11T20:36:07,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742255_1431 (size=12301) 2024-12-11T20:36:07,173 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T20:36:07,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-11T20:36:07,400 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/ce8a8556e645423cb2aa6ebc5f170db7 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/ce8a8556e645423cb2aa6ebc5f170db7 2024-12-11T20:36:07,404 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/A of 4451ee260f37d1215790a309e039ba65 into ce8a8556e645423cb2aa6ebc5f170db7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:07,404 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:07,404 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/A, priority=13, startTime=1733949366476; duration=0sec 2024-12-11T20:36:07,404 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:07,404 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:A 2024-12-11T20:36:07,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:07,455 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/ef3dac9f3828410da378e66cb29c4452 2024-12-11T20:36:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/c4bc88f6a052477582ea2f77fbbff802 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/c4bc88f6a052477582ea2f77fbbff802 2024-12-11T20:36:07,461 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/c4bc88f6a052477582ea2f77fbbff802, entries=150, sequenceid=315, filesize=30.5 K 2024-12-11T20:36:07,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/23cfd0320ca245c29c3da2bd7e58aa74 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/23cfd0320ca245c29c3da2bd7e58aa74 2024-12-11T20:36:07,464 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/23cfd0320ca245c29c3da2bd7e58aa74, entries=150, sequenceid=315, filesize=12.0 K 2024-12-11T20:36:07,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/ef3dac9f3828410da378e66cb29c4452 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/ef3dac9f3828410da378e66cb29c4452 2024-12-11T20:36:07,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949427464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,469 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/ef3dac9f3828410da378e66cb29c4452, entries=150, sequenceid=315, filesize=12.0 K 2024-12-11T20:36:07,469 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=67.09 KB/68700 for 4451ee260f37d1215790a309e039ba65 in 988ms, sequenceid=315, compaction requested=false 2024-12-11T20:36:07,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:07,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:07,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-11T20:36:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-11T20:36:07,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-11T20:36:07,472 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3700 sec 2024-12-11T20:36:07,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.3740 sec 2024-12-11T20:36:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:07,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-11T20:36:07,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:36:07,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:07,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:36:07,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:07,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:36:07,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:07,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115086052b18dc443eb2b4459ef8d9987f_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949367473/Put/seqid=0 2024-12-11T20:36:07,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742256_1432 (size=17534) 2024-12-11T20:36:07,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949427495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949427500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949427500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949427502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949427577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949427606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949427617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949427617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949427617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949427787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949427816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949427827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949427827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949427827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:07,888 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:07,894 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115086052b18dc443eb2b4459ef8d9987f_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115086052b18dc443eb2b4459ef8d9987f_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:07,895 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/b973d81fe4a24ea3afe3775c7fa48825, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:07,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/b973d81fe4a24ea3afe3775c7fa48825 is 175, key is test_row_0/A:col10/1733949367473/Put/seqid=0 2024-12-11T20:36:07,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742257_1433 (size=48639) 2024-12-11T20:36:08,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:08,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949428097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:08,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:08,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949428126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:08,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:08,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949428137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:08,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949428137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:08,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949428137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:08,299 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/b973d81fe4a24ea3afe3775c7fa48825 2024-12-11T20:36:08,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/2af709021c70485ea8d1b17a32a673a7 is 50, key is test_row_0/B:col10/1733949367473/Put/seqid=0 2024-12-11T20:36:08,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742258_1434 (size=12301) 2024-12-11T20:36:08,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:08,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949428606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:08,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949428637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:08,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949428647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:08,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:08,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949428647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:08,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:08,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:08,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949428656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:08,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/2af709021c70485ea8d1b17a32a673a7 2024-12-11T20:36:08,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/29918b8897d743a6830efabf492b3176 is 50, key is test_row_0/C:col10/1733949367473/Put/seqid=0 2024-12-11T20:36:08,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742259_1435 (size=12301) 2024-12-11T20:36:09,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/29918b8897d743a6830efabf492b3176 2024-12-11T20:36:09,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/b973d81fe4a24ea3afe3775c7fa48825 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b973d81fe4a24ea3afe3775c7fa48825 2024-12-11T20:36:09,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b973d81fe4a24ea3afe3775c7fa48825, entries=250, sequenceid=333, filesize=47.5 K 2024-12-11T20:36:09,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/2af709021c70485ea8d1b17a32a673a7 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2af709021c70485ea8d1b17a32a673a7 2024-12-11T20:36:09,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2af709021c70485ea8d1b17a32a673a7, entries=150, sequenceid=333, filesize=12.0 K 2024-12-11T20:36:09,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/29918b8897d743a6830efabf492b3176 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/29918b8897d743a6830efabf492b3176 2024-12-11T20:36:09,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/29918b8897d743a6830efabf492b3176, entries=150, sequenceid=333, filesize=12.0 K 2024-12-11T20:36:09,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4451ee260f37d1215790a309e039ba65 in 1663ms, sequenceid=333, compaction requested=true 2024-12-11T20:36:09,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:09,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:09,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:09,138 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:09,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:09,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:09,138 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:09,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:09,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:09,138 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:09,138 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111820 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:09,138 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/B is initiating minor compaction (all files) 2024-12-11T20:36:09,138 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/A is initiating minor compaction (all files) 2024-12-11T20:36:09,138 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/B in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:09,139 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/A in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:09,139 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/63deb2e45b0548c4a602a45c8f8fadf5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/23cfd0320ca245c29c3da2bd7e58aa74, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2af709021c70485ea8d1b17a32a673a7] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=36.6 K 2024-12-11T20:36:09,139 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/ce8a8556e645423cb2aa6ebc5f170db7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/c4bc88f6a052477582ea2f77fbbff802, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b973d81fe4a24ea3afe3775c7fa48825] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=109.2 K 2024-12-11T20:36:09,139 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:09,139 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/ce8a8556e645423cb2aa6ebc5f170db7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/c4bc88f6a052477582ea2f77fbbff802, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b973d81fe4a24ea3afe3775c7fa48825] 2024-12-11T20:36:09,139 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 63deb2e45b0548c4a602a45c8f8fadf5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733949362839 2024-12-11T20:36:09,139 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 23cfd0320ca245c29c3da2bd7e58aa74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733949365240 2024-12-11T20:36:09,139 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce8a8556e645423cb2aa6ebc5f170db7, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733949362839 2024-12-11T20:36:09,139 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 2af709021c70485ea8d1b17a32a673a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733949367465 2024-12-11T20:36:09,139 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4bc88f6a052477582ea2f77fbbff802, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733949365240 2024-12-11T20:36:09,140 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b973d81fe4a24ea3afe3775c7fa48825, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733949367454 2024-12-11T20:36:09,144 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#B#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:09,144 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/fa8942e9b3d24c71b7d28c1deefae98a is 50, key is test_row_0/B:col10/1733949367473/Put/seqid=0 2024-12-11T20:36:09,147 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:09,149 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211cbff6eff482a427ea004059bbb7afe28_4451ee260f37d1215790a309e039ba65 store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:09,150 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211cbff6eff482a427ea004059bbb7afe28_4451ee260f37d1215790a309e039ba65, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:09,150 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211cbff6eff482a427ea004059bbb7afe28_4451ee260f37d1215790a309e039ba65 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:09,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742260_1436 (size=13017) 2024-12-11T20:36:09,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742261_1437 (size=4469) 2024-12-11T20:36:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-11T20:36:09,267 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:09,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-11T20:36:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-11T20:36:09,270 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:09,270 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:09,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-11T20:36:09,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-11T20:36:09,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:09,423 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T20:36:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:36:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:36:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:36:09,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:09,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211beb8db7844c24198ba895dce1d9f0292_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949367499/Put/seqid=0 2024-12-11T20:36:09,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742262_1438 (size=12454) 2024-12-11T20:36:09,556 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/fa8942e9b3d24c71b7d28c1deefae98a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fa8942e9b3d24c71b7d28c1deefae98a 2024-12-11T20:36:09,557 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#A#compaction#366 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:09,557 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/96008350cebf40d9819e5f7eeefbd536 is 175, key is test_row_0/A:col10/1733949367473/Put/seqid=0 2024-12-11T20:36:09,560 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/B of 4451ee260f37d1215790a309e039ba65 into fa8942e9b3d24c71b7d28c1deefae98a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:09,560 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:09,560 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/B, priority=13, startTime=1733949369138; duration=0sec 2024-12-11T20:36:09,560 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:09,560 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:B 2024-12-11T20:36:09,560 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:09,561 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:09,561 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): 4451ee260f37d1215790a309e039ba65/C is initiating minor compaction (all files) 2024-12-11T20:36:09,561 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4451ee260f37d1215790a309e039ba65/C in TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:09,561 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/2de41a7ce6444121aad046a46c441990, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/ef3dac9f3828410da378e66cb29c4452, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/29918b8897d743a6830efabf492b3176] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp, totalSize=36.6 K 2024-12-11T20:36:09,562 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 2de41a7ce6444121aad046a46c441990, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1733949362839 2024-12-11T20:36:09,562 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting ef3dac9f3828410da378e66cb29c4452, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733949365240 2024-12-11T20:36:09,562 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 29918b8897d743a6830efabf492b3176, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1733949367465 2024-12-11T20:36:09,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742263_1439 (size=31971) 2024-12-11T20:36:09,568 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4451ee260f37d1215790a309e039ba65#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:09,568 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/24cbb79c2b13403589258ad6bdc775e0 is 50, key is test_row_0/C:col10/1733949367473/Put/seqid=0 2024-12-11T20:36:09,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742264_1440 (size=13017) 2024-12-11T20:36:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-11T20:36:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:09,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. as already flushing 2024-12-11T20:36:09,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57970 deadline: 1733949429668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:09,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57948 deadline: 1733949429668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:09,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57964 deadline: 1733949429668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:09,672 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in 4451ee260f37d1215790a309e039ba65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:09,672 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_0' in 4451ee260f37d1215790a309e039ba65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,672 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in 4451ee260f37d1215790a309e039ba65 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:09,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949429671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:09,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:09,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949429672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:09,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949429787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:09,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:09,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949429787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:09,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:09,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:09,836 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211beb8db7844c24198ba895dce1d9f0292_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211beb8db7844c24198ba895dce1d9f0292_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:09,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/e5e2d56fe20940ccaf85d4e223b73730, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:09,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/e5e2d56fe20940ccaf85d4e223b73730 is 175, key is test_row_0/A:col10/1733949367499/Put/seqid=0 2024-12-11T20:36:09,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742265_1441 (size=31255) 2024-12-11T20:36:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-11T20:36:09,968 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/96008350cebf40d9819e5f7eeefbd536 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/96008350cebf40d9819e5f7eeefbd536 2024-12-11T20:36:09,971 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/A of 4451ee260f37d1215790a309e039ba65 into 96008350cebf40d9819e5f7eeefbd536(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:09,972 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:09,972 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/A, priority=13, startTime=1733949369138; duration=0sec 2024-12-11T20:36:09,972 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:09,972 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:A 2024-12-11T20:36:09,975 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/24cbb79c2b13403589258ad6bdc775e0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/24cbb79c2b13403589258ad6bdc775e0 2024-12-11T20:36:09,978 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4451ee260f37d1215790a309e039ba65/C of 4451ee260f37d1215790a309e039ba65 into 24cbb79c2b13403589258ad6bdc775e0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:09,978 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:09,978 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., storeName=4451ee260f37d1215790a309e039ba65/C, priority=13, startTime=1733949369138; duration=0sec 2024-12-11T20:36:09,978 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:09,978 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:C 2024-12-11T20:36:10,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949429997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949429997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:10,153 INFO [Thread-1615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1615. 2024-12-11T20:36:10,153 DEBUG [Thread-1615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:10,153 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:10,153 DEBUG [Thread-1615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:10,153 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:10,155 INFO [Thread-1611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1611. 2024-12-11T20:36:10,155 DEBUG [Thread-1611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:10,155 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:10,155 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:10,155 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:10,159 INFO [Thread-1609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1609. 2024-12-11T20:36:10,159 DEBUG [Thread-1609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:10,159 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:10,159 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:10,159 INFO [Thread-1613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1613. 2024-12-11T20:36:10,159 DEBUG [Thread-1613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:10,159 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:10,159 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:10,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:10,159 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:10,161 INFO [Thread-1617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1617. 2024-12-11T20:36:10,161 DEBUG [Thread-1617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:10,161 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:10,161 DEBUG [Thread-1617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:10,161 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:10,241 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/e5e2d56fe20940ccaf85d4e223b73730 2024-12-11T20:36:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/705c3b680be543c58d08f6d3c7af7715 is 50, key is test_row_0/B:col10/1733949367499/Put/seqid=0 2024-12-11T20:36:10,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742266_1442 (size=12301) 2024-12-11T20:36:10,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:10,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949430317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949430317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-11T20:36:10,651 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/705c3b680be543c58d08f6d3c7af7715 2024-12-11T20:36:10,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/ea6f1c68cc564704bce5973ed8613682 is 50, key is test_row_0/C:col10/1733949367499/Put/seqid=0 2024-12-11T20:36:10,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742267_1443 (size=12301) 2024-12-11T20:36:10,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:10,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57900 deadline: 1733949430827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:10,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57934 deadline: 1733949430827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:11,064 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/ea6f1c68cc564704bce5973ed8613682 2024-12-11T20:36:11,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/e5e2d56fe20940ccaf85d4e223b73730 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/e5e2d56fe20940ccaf85d4e223b73730 2024-12-11T20:36:11,074 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/e5e2d56fe20940ccaf85d4e223b73730, entries=150, sequenceid=350, filesize=30.5 K 2024-12-11T20:36:11,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/705c3b680be543c58d08f6d3c7af7715 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/705c3b680be543c58d08f6d3c7af7715 2024-12-11T20:36:11,079 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/705c3b680be543c58d08f6d3c7af7715, entries=150, sequenceid=350, filesize=12.0 K 2024-12-11T20:36:11,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/ea6f1c68cc564704bce5973ed8613682 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/ea6f1c68cc564704bce5973ed8613682 2024-12-11T20:36:11,083 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/ea6f1c68cc564704bce5973ed8613682, entries=150, sequenceid=350, filesize=12.0 K 2024-12-11T20:36:11,084 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 4451ee260f37d1215790a309e039ba65 in 1662ms, sequenceid=350, compaction requested=false 2024-12-11T20:36:11,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:11,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:11,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-11T20:36:11,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-11T20:36:11,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-11T20:36:11,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-12-11T20:36:11,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8180 sec 2024-12-11T20:36:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-11T20:36:11,419 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on 4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:11,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:36:11,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:36:11,682 INFO [Thread-1604 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1604. 2024-12-11T20:36:11,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:11,682 DEBUG [Thread-1604 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:11,682 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:36:11,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:11,682 DEBUG [Thread-1604 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:36:11,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:11,682 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:11,688 INFO [Thread-1606 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1606. 2024-12-11T20:36:11,688 INFO [Thread-1598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1598. 2024-12-11T20:36:11,688 DEBUG [Thread-1606 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:11,688 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,688 DEBUG [Thread-1598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:11,688 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,688 DEBUG [Thread-1606 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,689 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:11,689 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:11,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211b9531c601b534192a9011d2f1b778362_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949369669/Put/seqid=0 2024-12-11T20:36:11,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742268_1444 (size=12454) 2024-12-11T20:36:11,792 WARN [regionserver/5be53b084ac7:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 1 2024-12-11T20:36:11,798 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5be53b084ac7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/128d6c41763b4459adae166b279e0f7f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/512d51a2a7154ee49032c08fc4eeea24, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b75310c2aef14eacb5d1cae26a59437e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/6886523d0cec4d69b86bdb12dffc02ab, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/d472ac5a154a46989aca60e4d3c5c5e2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/556d66f5ef7241dd96c17f2e10d59eb2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/5ecd12e6e8634d6fa18fa00eb737607c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/261f6aa66691495190d0be2267396774, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/48e6aa56e35244548b9a08a7621cd89b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/7bcddfaead6f44bdb09a870eb83b291a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b51498c9596142c7a34a879a365c4066, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/705dbd1dc56c4f6eb9607d2ce62ca1f0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/aeb90e10a95b4ee0b35ba8bf544a2fef, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/294c954df49b4d3c938675e14d7a8f15, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/1f5f019574424597a54e98379a072411, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/bf9796b592f442338985120ca8bc624a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/9ee8fa2610604885bc64add7c8415ae7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/ce8a8556e645423cb2aa6ebc5f170db7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/78eae041fb5641fbbc8e3d63c2331707, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/c4bc88f6a052477582ea2f77fbbff802, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b973d81fe4a24ea3afe3775c7fa48825] to archive 2024-12-11T20:36:11,799 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5be53b084ac7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:36:11,801 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/128d6c41763b4459adae166b279e0f7f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/128d6c41763b4459adae166b279e0f7f 2024-12-11T20:36:11,801 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b75310c2aef14eacb5d1cae26a59437e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b75310c2aef14eacb5d1cae26a59437e 2024-12-11T20:36:11,801 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/556d66f5ef7241dd96c17f2e10d59eb2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/556d66f5ef7241dd96c17f2e10d59eb2 2024-12-11T20:36:11,801 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/6886523d0cec4d69b86bdb12dffc02ab to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/6886523d0cec4d69b86bdb12dffc02ab 2024-12-11T20:36:11,801 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/d472ac5a154a46989aca60e4d3c5c5e2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/d472ac5a154a46989aca60e4d3c5c5e2 2024-12-11T20:36:11,801 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/512d51a2a7154ee49032c08fc4eeea24 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/512d51a2a7154ee49032c08fc4eeea24 2024-12-11T20:36:11,801 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/261f6aa66691495190d0be2267396774 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/261f6aa66691495190d0be2267396774 2024-12-11T20:36:11,801 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/5ecd12e6e8634d6fa18fa00eb737607c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/5ecd12e6e8634d6fa18fa00eb737607c 2024-12-11T20:36:11,802 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/7bcddfaead6f44bdb09a870eb83b291a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/7bcddfaead6f44bdb09a870eb83b291a 2024-12-11T20:36:11,802 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/48e6aa56e35244548b9a08a7621cd89b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/48e6aa56e35244548b9a08a7621cd89b 2024-12-11T20:36:11,802 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/aeb90e10a95b4ee0b35ba8bf544a2fef to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/aeb90e10a95b4ee0b35ba8bf544a2fef 2024-12-11T20:36:11,802 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/294c954df49b4d3c938675e14d7a8f15 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/294c954df49b4d3c938675e14d7a8f15 2024-12-11T20:36:11,802 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/1f5f019574424597a54e98379a072411 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/1f5f019574424597a54e98379a072411 2024-12-11T20:36:11,802 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b51498c9596142c7a34a879a365c4066 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b51498c9596142c7a34a879a365c4066 2024-12-11T20:36:11,803 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/bf9796b592f442338985120ca8bc624a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/bf9796b592f442338985120ca8bc624a 2024-12-11T20:36:11,803 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/705dbd1dc56c4f6eb9607d2ce62ca1f0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/705dbd1dc56c4f6eb9607d2ce62ca1f0 2024-12-11T20:36:11,803 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/9ee8fa2610604885bc64add7c8415ae7 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/9ee8fa2610604885bc64add7c8415ae7 2024-12-11T20:36:11,803 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/ce8a8556e645423cb2aa6ebc5f170db7 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/ce8a8556e645423cb2aa6ebc5f170db7 2024-12-11T20:36:11,803 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/78eae041fb5641fbbc8e3d63c2331707 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/78eae041fb5641fbbc8e3d63c2331707 2024-12-11T20:36:11,803 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/c4bc88f6a052477582ea2f77fbbff802 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/c4bc88f6a052477582ea2f77fbbff802 2024-12-11T20:36:11,804 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b973d81fe4a24ea3afe3775c7fa48825 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/b973d81fe4a24ea3afe3775c7fa48825 2024-12-11T20:36:11,806 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5be53b084ac7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e2b1b09406c646eb8c38f9848a0924f7, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/0e10cfdbeead4c4ba9660df4943720d0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/99265d1b05ee4764b761d5cada0c9de5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fe9fa87b5d144c74aa09560866e7e427, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e27fc3c4847c4a01bcaeef5fddd1999c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/ac4cc8ba6bac43a895575187a26b30df, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/f603557fe4764f1981fd2c6ac6802b5b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/ca65fa65d509440788592bc930cfc670, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/b7d5f306d9f54ed19b9db3b8208def93, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2aba3cd9ae6a4e3b853ab2e0c078235b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/1e86de5f9b154a119d7e5c46cefbc2a0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e37ae65d190148ef96cc192a4b57629f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fad0994532be43818f6b4c5cb1e04eae, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/7058db94df0347e6ae90ccac970b7d3d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/218f78eba8c04711990136d9dc7d9e7e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/33ce4b46b74848a389d38554179936db, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/3ef513c954fe44ec96b8771dd933ad6d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/63deb2e45b0548c4a602a45c8f8fadf5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/b517c84e2c724ca1b5d69ee9742290ba, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/23cfd0320ca245c29c3da2bd7e58aa74, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2af709021c70485ea8d1b17a32a673a7] to archive 2024-12-11T20:36:11,807 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5be53b084ac7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:36:11,808 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/99265d1b05ee4764b761d5cada0c9de5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/99265d1b05ee4764b761d5cada0c9de5 2024-12-11T20:36:11,808 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e2b1b09406c646eb8c38f9848a0924f7 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e2b1b09406c646eb8c38f9848a0924f7 2024-12-11T20:36:11,808 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/ac4cc8ba6bac43a895575187a26b30df to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/ac4cc8ba6bac43a895575187a26b30df 2024-12-11T20:36:11,808 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fe9fa87b5d144c74aa09560866e7e427 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fe9fa87b5d144c74aa09560866e7e427 2024-12-11T20:36:11,808 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/0e10cfdbeead4c4ba9660df4943720d0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/0e10cfdbeead4c4ba9660df4943720d0 2024-12-11T20:36:11,809 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e27fc3c4847c4a01bcaeef5fddd1999c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e27fc3c4847c4a01bcaeef5fddd1999c 2024-12-11T20:36:11,809 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/ca65fa65d509440788592bc930cfc670 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/ca65fa65d509440788592bc930cfc670 2024-12-11T20:36:11,809 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/f603557fe4764f1981fd2c6ac6802b5b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/f603557fe4764f1981fd2c6ac6802b5b 2024-12-11T20:36:11,810 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/b7d5f306d9f54ed19b9db3b8208def93 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/b7d5f306d9f54ed19b9db3b8208def93 2024-12-11T20:36:11,810 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2aba3cd9ae6a4e3b853ab2e0c078235b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2aba3cd9ae6a4e3b853ab2e0c078235b 2024-12-11T20:36:11,810 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fad0994532be43818f6b4c5cb1e04eae to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fad0994532be43818f6b4c5cb1e04eae 2024-12-11T20:36:11,810 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e37ae65d190148ef96cc192a4b57629f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/e37ae65d190148ef96cc192a4b57629f 2024-12-11T20:36:11,810 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/7058db94df0347e6ae90ccac970b7d3d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/7058db94df0347e6ae90ccac970b7d3d 2024-12-11T20:36:11,810 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/218f78eba8c04711990136d9dc7d9e7e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/218f78eba8c04711990136d9dc7d9e7e 2024-12-11T20:36:11,810 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/1e86de5f9b154a119d7e5c46cefbc2a0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/1e86de5f9b154a119d7e5c46cefbc2a0 2024-12-11T20:36:11,810 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/33ce4b46b74848a389d38554179936db to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/33ce4b46b74848a389d38554179936db 2024-12-11T20:36:11,811 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/3ef513c954fe44ec96b8771dd933ad6d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/3ef513c954fe44ec96b8771dd933ad6d 2024-12-11T20:36:11,811 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/63deb2e45b0548c4a602a45c8f8fadf5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/63deb2e45b0548c4a602a45c8f8fadf5 2024-12-11T20:36:11,811 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/b517c84e2c724ca1b5d69ee9742290ba to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/b517c84e2c724ca1b5d69ee9742290ba 2024-12-11T20:36:11,811 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2af709021c70485ea8d1b17a32a673a7 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2af709021c70485ea8d1b17a32a673a7 2024-12-11T20:36:11,811 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/23cfd0320ca245c29c3da2bd7e58aa74 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/23cfd0320ca245c29c3da2bd7e58aa74 2024-12-11T20:36:11,814 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5be53b084ac7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6f3c52fc7beb4a7b8890f3a770e39e1b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/953eaa31117f433f8e256e23939f5f67, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/d0639cabd1974883b0e7e5ce98fc5773, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/40eb8d9212464a15892e1be918f06d53, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/7d844acf9d214531bbd4e325a572f82a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/f306523e15394f87afa842d3bccf8408, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6da6f9558f124e7490e067c3e6a171a5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/e6c79856a302439b810e4a30a2eacc26, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/93af30d6743f414b8e5928ca5fb6f81a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/363cd288e2f34797aa61465057a93c3e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4e9229ecf4e24c6f914eede897a66f6b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4c14053c3a734290a534f7edbaa6f7bc, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6c5da9c6fa844764bfa4fb7195ab4284, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/5cf5766e778b4ba183cf8a722da2146b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/d86ecb52f9694f3ba8a570f0b59a1125, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4ca042ab35924f8494f1a29d5663b47c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/3a0f6640913b424bac2c1a1efdb67a35, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/2de41a7ce6444121aad046a46c441990, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/bce74aed360b4337ad96c4431d1acf08, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/ef3dac9f3828410da378e66cb29c4452, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/29918b8897d743a6830efabf492b3176] to archive 2024-12-11T20:36:11,815 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5be53b084ac7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:36:11,816 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6f3c52fc7beb4a7b8890f3a770e39e1b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6f3c52fc7beb4a7b8890f3a770e39e1b 2024-12-11T20:36:11,816 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/d0639cabd1974883b0e7e5ce98fc5773 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/d0639cabd1974883b0e7e5ce98fc5773 2024-12-11T20:36:11,816 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/953eaa31117f433f8e256e23939f5f67 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/953eaa31117f433f8e256e23939f5f67 2024-12-11T20:36:11,817 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/40eb8d9212464a15892e1be918f06d53 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/40eb8d9212464a15892e1be918f06d53 2024-12-11T20:36:11,817 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/e6c79856a302439b810e4a30a2eacc26 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/e6c79856a302439b810e4a30a2eacc26 2024-12-11T20:36:11,817 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6da6f9558f124e7490e067c3e6a171a5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6da6f9558f124e7490e067c3e6a171a5 2024-12-11T20:36:11,817 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/f306523e15394f87afa842d3bccf8408 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/f306523e15394f87afa842d3bccf8408 2024-12-11T20:36:11,817 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/7d844acf9d214531bbd4e325a572f82a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/7d844acf9d214531bbd4e325a572f82a 2024-12-11T20:36:11,818 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4e9229ecf4e24c6f914eede897a66f6b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4e9229ecf4e24c6f914eede897a66f6b 2024-12-11T20:36:11,818 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/93af30d6743f414b8e5928ca5fb6f81a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/93af30d6743f414b8e5928ca5fb6f81a 2024-12-11T20:36:11,818 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6c5da9c6fa844764bfa4fb7195ab4284 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/6c5da9c6fa844764bfa4fb7195ab4284 2024-12-11T20:36:11,818 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4c14053c3a734290a534f7edbaa6f7bc to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4c14053c3a734290a534f7edbaa6f7bc 2024-12-11T20:36:11,818 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/363cd288e2f34797aa61465057a93c3e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/363cd288e2f34797aa61465057a93c3e 2024-12-11T20:36:11,819 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/5cf5766e778b4ba183cf8a722da2146b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/5cf5766e778b4ba183cf8a722da2146b 2024-12-11T20:36:11,819 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/d86ecb52f9694f3ba8a570f0b59a1125 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/d86ecb52f9694f3ba8a570f0b59a1125 2024-12-11T20:36:11,819 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4ca042ab35924f8494f1a29d5663b47c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/4ca042ab35924f8494f1a29d5663b47c 2024-12-11T20:36:11,820 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/3a0f6640913b424bac2c1a1efdb67a35 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/3a0f6640913b424bac2c1a1efdb67a35 2024-12-11T20:36:11,820 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/bce74aed360b4337ad96c4431d1acf08 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/bce74aed360b4337ad96c4431d1acf08 2024-12-11T20:36:11,820 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/2de41a7ce6444121aad046a46c441990 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/2de41a7ce6444121aad046a46c441990 2024-12-11T20:36:11,820 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/29918b8897d743a6830efabf492b3176 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/29918b8897d743a6830efabf492b3176 2024-12-11T20:36:11,820 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/ef3dac9f3828410da378e66cb29c4452 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/ef3dac9f3828410da378e66cb29c4452 2024-12-11T20:36:11,839 INFO [Thread-1600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1600. 2024-12-11T20:36:11,839 DEBUG [Thread-1600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:11,839 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,839 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:11,848 INFO [Thread-1602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1602. 2024-12-11T20:36:11,848 DEBUG [Thread-1602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:11,848 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,848 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1354 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4061 rows 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1356 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4061 rows 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1369 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4103 rows 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1360 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4073 rows 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1368 2024-12-11T20:36:11,848 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4103 rows 2024-12-11T20:36:11,848 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T20:36:11,848 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:36:11,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:11,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:11,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-11T20:36:11,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:11,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T20:36:11,852 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949371852"}]},"ts":"1733949371852"} 2024-12-11T20:36:11,854 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T20:36:11,854 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T20:36:11,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T20:36:11,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, UNASSIGN}] 2024-12-11T20:36:11,857 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, UNASSIGN 2024-12-11T20:36:11,858 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=4451ee260f37d1215790a309e039ba65, regionState=CLOSING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:11,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, UNASSIGN because future has completed 2024-12-11T20:36:11,859 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T20:36:11,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:36:11,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T20:36:12,013 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:12,013 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T20:36:12,013 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 4451ee260f37d1215790a309e039ba65, disabling compactions & flushes 2024-12-11T20:36:12,014 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:12,097 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:12,104 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211b9531c601b534192a9011d2f1b778362_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b9531c601b534192a9011d2f1b778362_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:12,105 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/de668840ed674143b136ba7aa46fb5f8, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:12,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/de668840ed674143b136ba7aa46fb5f8 is 175, key is test_row_0/A:col10/1733949369669/Put/seqid=0 2024-12-11T20:36:12,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742269_1445 (size=31255) 2024-12-11T20:36:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T20:36:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T20:36:12,510 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/de668840ed674143b136ba7aa46fb5f8 2024-12-11T20:36:12,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/a561f3c4ddb044b89a14cc2641c9aa75 is 50, key is test_row_0/B:col10/1733949369669/Put/seqid=0 2024-12-11T20:36:12,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742270_1446 (size=12301) 2024-12-11T20:36:12,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/a561f3c4ddb044b89a14cc2641c9aa75 2024-12-11T20:36:12,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/312001dfc1f747a58de711d066568d35 is 50, key is test_row_0/C:col10/1733949369669/Put/seqid=0 2024-12-11T20:36:12,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742271_1447 (size=12301) 2024-12-11T20:36:12,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T20:36:13,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/312001dfc1f747a58de711d066568d35 2024-12-11T20:36:13,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/de668840ed674143b136ba7aa46fb5f8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/de668840ed674143b136ba7aa46fb5f8 2024-12-11T20:36:13,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/de668840ed674143b136ba7aa46fb5f8, entries=150, sequenceid=369, filesize=30.5 K 2024-12-11T20:36:13,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/a561f3c4ddb044b89a14cc2641c9aa75 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/a561f3c4ddb044b89a14cc2641c9aa75 2024-12-11T20:36:13,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/a561f3c4ddb044b89a14cc2641c9aa75, entries=150, sequenceid=369, filesize=12.0 K 2024-12-11T20:36:13,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/312001dfc1f747a58de711d066568d35 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/312001dfc1f747a58de711d066568d35 2024-12-11T20:36:13,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/312001dfc1f747a58de711d066568d35, entries=150, sequenceid=369, filesize=12.0 K 2024-12-11T20:36:13,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=26.84 KB/27480 for 4451ee260f37d1215790a309e039ba65 in 1670ms, sequenceid=369, compaction requested=true 2024-12-11T20:36:13,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4451ee260f37d1215790a309e039ba65: 2024-12-11T20:36:13,351 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:13,351 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:13,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:13,351 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. because compaction request was cancelled 2024-12-11T20:36:13,351 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. after waiting 0 ms 2024-12-11T20:36:13,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:13,351 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:A 2024-12-11T20:36:13,351 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:13,351 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. because compaction request was cancelled 2024-12-11T20:36:13,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:B, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:13,351 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:B 2024-12-11T20:36:13,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:13,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4451ee260f37d1215790a309e039ba65:C, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:13,351 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 4451ee260f37d1215790a309e039ba65 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-11T20:36:13,351 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. because compaction request was cancelled 2024-12-11T20:36:13,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:13,351 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4451ee260f37d1215790a309e039ba65:C 2024-12-11T20:36:13,351 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=A 2024-12-11T20:36:13,351 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:13,351 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=B 2024-12-11T20:36:13,351 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:13,351 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4451ee260f37d1215790a309e039ba65, store=C 2024-12-11T20:36:13,351 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:13,355 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bfa3b5a3468540a28533a405e8a1ad5d_4451ee260f37d1215790a309e039ba65 is 50, key is test_row_0/A:col10/1733949371687/Put/seqid=0 2024-12-11T20:36:13,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742272_1448 (size=12454) 2024-12-11T20:36:13,760 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:13,763 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bfa3b5a3468540a28533a405e8a1ad5d_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bfa3b5a3468540a28533a405e8a1ad5d_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:13,764 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/eff86f3d40c44d73afea0e08e7cd89c9, store: [table=TestAcidGuarantees family=A region=4451ee260f37d1215790a309e039ba65] 2024-12-11T20:36:13,764 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/eff86f3d40c44d73afea0e08e7cd89c9 is 175, key is test_row_0/A:col10/1733949371687/Put/seqid=0 2024-12-11T20:36:13,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742273_1449 (size=31255) 2024-12-11T20:36:13,768 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=376, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/eff86f3d40c44d73afea0e08e7cd89c9 2024-12-11T20:36:13,773 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/2a00a360022f4b9ca993584b5bb0ed8b is 50, key is test_row_0/B:col10/1733949371687/Put/seqid=0 2024-12-11T20:36:13,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742274_1450 (size=12301) 2024-12-11T20:36:13,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T20:36:14,177 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/2a00a360022f4b9ca993584b5bb0ed8b 2024-12-11T20:36:14,182 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/e97ce79bb9f0485ababbb9393b2542ed is 50, key is test_row_0/C:col10/1733949371687/Put/seqid=0 2024-12-11T20:36:14,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742275_1451 (size=12301) 2024-12-11T20:36:14,586 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/e97ce79bb9f0485ababbb9393b2542ed 2024-12-11T20:36:14,589 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/A/eff86f3d40c44d73afea0e08e7cd89c9 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/eff86f3d40c44d73afea0e08e7cd89c9 2024-12-11T20:36:14,592 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/eff86f3d40c44d73afea0e08e7cd89c9, entries=150, sequenceid=376, filesize=30.5 K 2024-12-11T20:36:14,593 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/B/2a00a360022f4b9ca993584b5bb0ed8b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2a00a360022f4b9ca993584b5bb0ed8b 2024-12-11T20:36:14,596 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2a00a360022f4b9ca993584b5bb0ed8b, entries=150, sequenceid=376, filesize=12.0 K 2024-12-11T20:36:14,596 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/.tmp/C/e97ce79bb9f0485ababbb9393b2542ed as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/e97ce79bb9f0485ababbb9393b2542ed 2024-12-11T20:36:14,599 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/e97ce79bb9f0485ababbb9393b2542ed, entries=150, sequenceid=376, filesize=12.0 K 2024-12-11T20:36:14,600 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4451ee260f37d1215790a309e039ba65 in 1249ms, sequenceid=376, compaction requested=true 2024-12-11T20:36:14,603 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/recovered.edits/379.seqid, newMaxSeqId=379, maxSeqId=4 2024-12-11T20:36:14,604 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. 2024-12-11T20:36:14,604 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 4451ee260f37d1215790a309e039ba65: Waiting for close lock at 1733949372013Running coprocessor pre-close hooks at 1733949372013Disabling compacts and flushes for region at 1733949372013Disabling writes for close at 1733949373351 (+1338 ms)Obtaining lock to block concurrent updates at 1733949373351Preparing flush snapshotting stores in 4451ee260f37d1215790a309e039ba65 at 1733949373351Finished memstore snapshotting TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733949373351Flushing stores of TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65. at 1733949373352 (+1 ms)Flushing 4451ee260f37d1215790a309e039ba65/A: creating writer at 1733949373352Flushing mob file 4451ee260f37d1215790a309e039ba65/A: appending metadata at 1733949373355 (+3 ms)Flushing mob file 4451ee260f37d1215790a309e039ba65/A: closing flushed file at 1733949373355Flushing 4451ee260f37d1215790a309e039ba65/A: appending metadata at 1733949373764 (+409 ms)Flushing 4451ee260f37d1215790a309e039ba65/A: closing flushed file at 1733949373764Flushing 4451ee260f37d1215790a309e039ba65/B: creating writer at 1733949373771 (+7 ms)Flushing 4451ee260f37d1215790a309e039ba65/B: appending metadata at 1733949373773 (+2 ms)Flushing 4451ee260f37d1215790a309e039ba65/B: closing flushed file at 1733949373773Flushing 4451ee260f37d1215790a309e039ba65/C: creating writer at 1733949374180 (+407 ms)Flushing 4451ee260f37d1215790a309e039ba65/C: appending metadata at 1733949374182 (+2 ms)Flushing 4451ee260f37d1215790a309e039ba65/C: closing flushed file at 1733949374182Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67c98dc3: reopening flushed file at 1733949374589 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5230b71: reopening flushed file at 1733949374592 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c899130: reopening flushed file at 1733949374596 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4451ee260f37d1215790a309e039ba65 in 1249ms, sequenceid=376, compaction requested=true at 1733949374600 (+4 ms)Writing region close event to WAL at 1733949374601 (+1 ms)Running coprocessor post-close hooks at 1733949374604 (+3 ms)Closed at 1733949374604 2024-12-11T20:36:14,606 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:14,606 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=4451ee260f37d1215790a309e039ba65, regionState=CLOSED 2024-12-11T20:36:14,608 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:36:14,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-11T20:36:14,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 4451ee260f37d1215790a309e039ba65, server=5be53b084ac7,44543,1733949248752 in 2.7490 sec 2024-12-11T20:36:14,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-11T20:36:14,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4451ee260f37d1215790a309e039ba65, UNASSIGN in 2.7540 sec 2024-12-11T20:36:14,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-11T20:36:14,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.7570 sec 2024-12-11T20:36:14,614 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949374614"}]},"ts":"1733949374614"} 2024-12-11T20:36:14,615 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T20:36:14,615 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T20:36:14,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.7670 sec 2024-12-11T20:36:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-11T20:36:16,019 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:16,020 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-11T20:36:16,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:16,025 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-11T20:36:16,026 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:16,028 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,031 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/recovered.edits] 2024-12-11T20:36:16,035 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/96008350cebf40d9819e5f7eeefbd536 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/96008350cebf40d9819e5f7eeefbd536 2024-12-11T20:36:16,035 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/de668840ed674143b136ba7aa46fb5f8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/de668840ed674143b136ba7aa46fb5f8 2024-12-11T20:36:16,035 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/eff86f3d40c44d73afea0e08e7cd89c9 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/eff86f3d40c44d73afea0e08e7cd89c9 2024-12-11T20:36:16,035 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/e5e2d56fe20940ccaf85d4e223b73730 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/A/e5e2d56fe20940ccaf85d4e223b73730 2024-12-11T20:36:16,039 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/705c3b680be543c58d08f6d3c7af7715 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/705c3b680be543c58d08f6d3c7af7715 2024-12-11T20:36:16,039 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2a00a360022f4b9ca993584b5bb0ed8b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/2a00a360022f4b9ca993584b5bb0ed8b 2024-12-11T20:36:16,039 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fa8942e9b3d24c71b7d28c1deefae98a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/fa8942e9b3d24c71b7d28c1deefae98a 2024-12-11T20:36:16,039 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/a561f3c4ddb044b89a14cc2641c9aa75 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/B/a561f3c4ddb044b89a14cc2641c9aa75 2024-12-11T20:36:16,041 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/24cbb79c2b13403589258ad6bdc775e0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/24cbb79c2b13403589258ad6bdc775e0 2024-12-11T20:36:16,041 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/312001dfc1f747a58de711d066568d35 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/312001dfc1f747a58de711d066568d35 2024-12-11T20:36:16,041 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/ea6f1c68cc564704bce5973ed8613682 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/ea6f1c68cc564704bce5973ed8613682 2024-12-11T20:36:16,041 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/e97ce79bb9f0485ababbb9393b2542ed to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/C/e97ce79bb9f0485ababbb9393b2542ed 2024-12-11T20:36:16,044 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/recovered.edits/379.seqid to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65/recovered.edits/379.seqid 2024-12-11T20:36:16,044 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,045 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T20:36:16,045 DEBUG [PEWorker-3 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T20:36:16,046 DEBUG [PEWorker-3 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-11T20:36:16,051 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412111098638f2c9a46139fe57a41088198f6_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412111098638f2c9a46139fe57a41088198f6_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,051 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412111ba0cb0627cd414ab150b30e564d334f_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412111ba0cb0627cd414ab150b30e564d334f_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,051 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121118ddef89d3e04db2ad05ad560815a74c_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121118ddef89d3e04db2ad05ad560815a74c_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,051 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114085cfd9bde14bdb966ba96359cf1c8c_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412114085cfd9bde14bdb966ba96359cf1c8c_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,051 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211406c3b2ccc334f30932a9ca6b6fced9d_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211406c3b2ccc334f30932a9ca6b6fced9d_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,051 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115086052b18dc443eb2b4459ef8d9987f_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115086052b18dc443eb2b4459ef8d9987f_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,052 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121128ca2bf5f7f64191ad3ad7f1c4313f66_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121128ca2bf5f7f64191ad3ad7f1c4313f66_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,052 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115b5d7103804a43789e79853cba9e18a1_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115b5d7103804a43789e79853cba9e18a1_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,052 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211710b528ea0ef4a3db208cb74ad90eb1c_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211710b528ea0ef4a3db208cb74ad90eb1c_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,052 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b95d8a5e92eb4bd58253f8283a469196_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b95d8a5e92eb4bd58253f8283a469196_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,052 DEBUG [HFileArchiver-23 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b9531c601b534192a9011d2f1b778362_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211b9531c601b534192a9011d2f1b778362_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,052 DEBUG [HFileArchiver-21 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211a0b4908fff1e448da85ce88674717c3e_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211a0b4908fff1e448da85ce88674717c3e_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,053 DEBUG [HFileArchiver-24 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412119a3e9612a1ee48768e07368ce0454869_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412119a3e9612a1ee48768e07368ce0454869_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,053 DEBUG [HFileArchiver-20 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211beb8db7844c24198ba895dce1d9f0292_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211beb8db7844c24198ba895dce1d9f0292_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,053 DEBUG [HFileArchiver-22 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bf2291fbd710468da3bcaa09393a9495_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bf2291fbd710468da3bcaa09393a9495_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,053 DEBUG [HFileArchiver-17 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bfa3b5a3468540a28533a405e8a1ad5d_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bfa3b5a3468540a28533a405e8a1ad5d_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,053 DEBUG [HFileArchiver-19 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211cad44166eeba459a9dcc91be69eae9dc_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211cad44166eeba459a9dcc91be69eae9dc_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,053 DEBUG [HFileArchiver-18 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211da46933363794c288af29612b03332c1_4451ee260f37d1215790a309e039ba65 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211da46933363794c288af29612b03332c1_4451ee260f37d1215790a309e039ba65 2024-12-11T20:36:16,054 DEBUG [PEWorker-3 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T20:36:16,055 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:16,057 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T20:36:16,060 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T20:36:16,061 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:16,061 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T20:36:16,061 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733949376061"}]},"ts":"9223372036854775807"} 2024-12-11T20:36:16,063 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T20:36:16,063 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4451ee260f37d1215790a309e039ba65, NAME => 'TestAcidGuarantees,,1733949347028.4451ee260f37d1215790a309e039ba65.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T20:36:16,063 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T20:36:16,063 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733949376063"}]},"ts":"9223372036854775807"} 2024-12-11T20:36:16,065 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T20:36:16,065 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:16,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-12-11T20:36:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-11T20:36:16,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T20:36:16,140 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:16,145 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=258 (was 253) - Thread LEAK? -, OpenFileDescriptor=457 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=391 (was 397), ProcessCount=11 (was 11), AvailableMemoryMB=1692 (was 1711) 2024-12-11T20:36:16,153 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=258, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=391, ProcessCount=11, AvailableMemoryMB=1691 2024-12-11T20:36:16,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T20:36:16,155 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:36:16,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:16,156 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:36:16,156 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:16,156 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-12-11T20:36:16,157 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:36:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-11T20:36:16,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742276_1452 (size=1061) 2024-12-11T20:36:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-11T20:36:16,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-11T20:36:16,563 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043 2024-12-11T20:36:16,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742277_1453 (size=53) 2024-12-11T20:36:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-11T20:36:16,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:36:16,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing da1a8ec404804dae6a302e6015dbd206, disabling compactions & flushes 2024-12-11T20:36:16,968 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:16,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:16,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. after waiting 0 ms 2024-12-11T20:36:16,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:16,968 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:16,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for da1a8ec404804dae6a302e6015dbd206: Waiting for close lock at 1733949376968Disabling compacts and flushes for region at 1733949376968Disabling writes for close at 1733949376968Writing region close event to WAL at 1733949376968Closed at 1733949376968 2024-12-11T20:36:16,970 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:36:16,970 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733949376970"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733949376970"}]},"ts":"1733949376970"} 2024-12-11T20:36:16,971 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:36:16,972 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:36:16,972 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949376972"}]},"ts":"1733949376972"} 2024-12-11T20:36:16,974 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T20:36:16,974 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da1a8ec404804dae6a302e6015dbd206, ASSIGN}] 2024-12-11T20:36:16,975 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da1a8ec404804dae6a302e6015dbd206, ASSIGN 2024-12-11T20:36:16,975 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da1a8ec404804dae6a302e6015dbd206, ASSIGN; state=OFFLINE, location=5be53b084ac7,44543,1733949248752; forceNewPlan=false, retain=false 2024-12-11T20:36:17,126 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=da1a8ec404804dae6a302e6015dbd206, regionState=OPENING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:17,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da1a8ec404804dae6a302e6015dbd206, ASSIGN because future has completed 2024-12-11T20:36:17,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:36:17,282 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:17,282 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:36:17,283 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,283 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:36:17,283 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,283 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,284 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,285 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:36:17,285 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da1a8ec404804dae6a302e6015dbd206 columnFamilyName A 2024-12-11T20:36:17,285 DEBUG [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:17,285 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] regionserver.HStore(327): Store=da1a8ec404804dae6a302e6015dbd206/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:36:17,285 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,286 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:36:17,286 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da1a8ec404804dae6a302e6015dbd206 columnFamilyName B 2024-12-11T20:36:17,286 DEBUG [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:17,287 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] regionserver.HStore(327): Store=da1a8ec404804dae6a302e6015dbd206/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:36:17,287 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,288 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:36:17,288 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region da1a8ec404804dae6a302e6015dbd206 columnFamilyName C 2024-12-11T20:36:17,288 DEBUG [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:17,288 INFO [StoreOpener-da1a8ec404804dae6a302e6015dbd206-1 {}] regionserver.HStore(327): Store=da1a8ec404804dae6a302e6015dbd206/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:36:17,288 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:17,288 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,289 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,289 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,290 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,290 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,290 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T20:36:17,291 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,292 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:36:17,293 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened da1a8ec404804dae6a302e6015dbd206; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61242278, jitterRate=-0.08741894364356995}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T20:36:17,293 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:17,293 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for da1a8ec404804dae6a302e6015dbd206: Running coprocessor pre-open hook at 1733949377283Writing region info on filesystem at 1733949377283Initializing all the Stores at 1733949377283Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949377283Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949377284 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949377284Cleaning up temporary data from old regions at 1733949377290 (+6 ms)Running coprocessor post-open hooks at 1733949377293 (+3 ms)Region opened successfully at 1733949377293 2024-12-11T20:36:17,294 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., pid=120, masterSystemTime=1733949377279 2024-12-11T20:36:17,295 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:17,295 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:17,296 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=da1a8ec404804dae6a302e6015dbd206, regionState=OPEN, openSeqNum=2, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-11T20:36:17,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:36:17,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-11T20:36:17,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 in 170 msec 2024-12-11T20:36:17,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-12-11T20:36:17,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da1a8ec404804dae6a302e6015dbd206, ASSIGN in 325 msec 2024-12-11T20:36:17,301 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:36:17,301 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949377301"}]},"ts":"1733949377301"} 2024-12-11T20:36:17,302 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T20:36:17,303 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:36:17,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-12-11T20:36:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-11T20:36:18,308 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:18,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@136fa33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:18,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:18,312 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b21c50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:18,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:18,316 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7892262d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,317 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:18,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:18,320 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:18,322 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T20:36:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 5be53b084ac7,39923,1733949248041 2024-12-11T20:36:18,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14c6d89e 2024-12-11T20:36:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T20:36:18,324 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60978, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T20:36:18,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a007a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:18,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:18,326 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c90ca85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:18,328 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:18,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3f77a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f50afe4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:18,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:18,331 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5516663, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,331 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:18,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:18,332 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:18,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e9db080, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e1f0eba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,334 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:18,334 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:18,334 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411f3aad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:18,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:18,336 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:18,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d769b01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@508a2eaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:18,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:18,338 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16cf705e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:18,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:18,339 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20a7e2e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b05efac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:18,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:18,342 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@259b23a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:18,342 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:18,343 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:18,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7da38739, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a57121, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:18,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:18,345 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:18,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c97b580, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:18,347 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23fd1be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,348 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@85d66cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:18,348 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:18,349 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65aae795, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:18,351 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:18,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6196f5cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643aa972, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:18,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:18,353 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d0ba117, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:18,354 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0baa15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ac3826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:18,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:18,356 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f26612d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:18,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:18,357 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:18,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7279dc88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14d41309, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:18,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:18,359 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@97595e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:18,360 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@499a3d65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:18,361 DEBUG [Thread-2052 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:18,361 DEBUG [Thread-2043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:18,361 DEBUG [Thread-2039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:18,361 DEBUG [Thread-2041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:18,361 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:18,362 DEBUG [Thread-2037 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:18,362 DEBUG [Thread-2046 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:18,363 DEBUG [Thread-2035 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:18,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:18,363 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:18,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-11T20:36:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:18,364 DEBUG [Thread-2048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:18,364 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:18,364 DEBUG [Thread-2054 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:18,364 DEBUG [Thread-2048 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:18,365 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:18,365 DEBUG [Thread-2048 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T20:36:18,365 DEBUG [Thread-2050 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:18,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:18,366 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:18,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:18,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:18,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:18,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:18,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:36:18,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:36:18,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:36:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:36:18,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41282, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:18,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:36:18,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:36:18,370 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:36:18,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:18,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:36:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:36:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2] 2024-12-11T20:36:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:18,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:36:18,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:18,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:18,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:18,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:18,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:18,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:18,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/9206b011e68e46569f9d99cd8ac4013d is 50, key is test_row_0/A:col10/1733949378372/Put/seqid=0 2024-12-11T20:36:18,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742278_1454 (size=14341) 2024-12-11T20:36:18,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/9206b011e68e46569f9d99cd8ac4013d 2024-12-11T20:36:18,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/78ff95b7401a4d97b4c4cf2039f09114 is 50, key is test_row_0/B:col10/1733949378372/Put/seqid=0 2024-12-11T20:36:18,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742279_1455 (size=12001) 2024-12-11T20:36:18,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949438403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949378390, max timestamp=1733949378441Num uniques 859; 2024-12-11T20:36:18,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949378390, max timestamp=1733949378441Num uniques 859; 2024-12-11T20:36:18,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949378390, max timestamp=1733949378441Num uniques 859; 2024-12-11T20:36:18,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:18,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:18,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T20:36:18,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41246 deadline: 1733949438482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41224 deadline: 1733949438482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1733949438488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41222 deadline: 1733949438490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-11T20:36:18,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:18,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:18,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:18,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:18,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:18,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:18,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949438547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41246 deadline: 1733949438597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41224 deadline: 1733949438597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1733949438606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41222 deadline: 1733949438606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-11T20:36:18,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:18,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:18,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:18,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:18,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T20:36:18,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949438757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41246 deadline: 1733949438807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41224 deadline: 1733949438807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1733949438817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-11T20:36:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:18,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:18,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41222 deadline: 1733949438817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:18,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:18,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:18,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/78ff95b7401a4d97b4c4cf2039f09114 2024-12-11T20:36:18,861 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/45e62147eada41a6b6094d59f8a72bea is 50, key is test_row_0/C:col10/1733949378372/Put/seqid=0 2024-12-11T20:36:18,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742280_1456 (size=12001) 2024-12-11T20:36:18,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/45e62147eada41a6b6094d59f8a72bea 2024-12-11T20:36:18,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/9206b011e68e46569f9d99cd8ac4013d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/9206b011e68e46569f9d99cd8ac4013d 2024-12-11T20:36:18,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/9206b011e68e46569f9d99cd8ac4013d, entries=200, sequenceid=12, filesize=14.0 K 2024-12-11T20:36:18,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/78ff95b7401a4d97b4c4cf2039f09114 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/78ff95b7401a4d97b4c4cf2039f09114 2024-12-11T20:36:18,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/78ff95b7401a4d97b4c4cf2039f09114, entries=150, sequenceid=12, filesize=11.7 K 2024-12-11T20:36:18,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/45e62147eada41a6b6094d59f8a72bea as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/45e62147eada41a6b6094d59f8a72bea 2024-12-11T20:36:18,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/45e62147eada41a6b6094d59f8a72bea, entries=150, sequenceid=12, filesize=11.7 K 2024-12-11T20:36:18,894 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for da1a8ec404804dae6a302e6015dbd206 in 516ms, sequenceid=12, compaction requested=false 2024-12-11T20:36:18,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:18,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-11T20:36:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:18,985 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-11T20:36:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:18,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:18,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/b16d7c754e4f49668899740ae0fa5723 is 50, key is test_row_0/A:col10/1733949378479/Put/seqid=0 2024-12-11T20:36:18,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742281_1457 (size=12001) 2024-12-11T20:36:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T20:36:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:19,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:19,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949439099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41246 deadline: 1733949439117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41224 deadline: 1733949439117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1733949439126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41222 deadline: 1733949439137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949439206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,215 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,394 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/b16d7c754e4f49668899740ae0fa5723 2024-12-11T20:36:19,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/8804f8fb42da40ee86a62cfb59abf18e is 50, key is test_row_0/B:col10/1733949378479/Put/seqid=0 2024-12-11T20:36:19,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742282_1458 (size=12001) 2024-12-11T20:36:19,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949439427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,431 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T20:36:19,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41246 deadline: 1733949439627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41224 deadline: 1733949439627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,631 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1733949439647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41222 deadline: 1733949439647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949439737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:19,804 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/8804f8fb42da40ee86a62cfb59abf18e 2024-12-11T20:36:19,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/3aa57af9592c466197db8c52e4b71c06 is 50, key is test_row_0/C:col10/1733949378479/Put/seqid=0 2024-12-11T20:36:19,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742283_1459 (size=12001) 2024-12-11T20:36:20,126 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:36:20,214 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/3aa57af9592c466197db8c52e4b71c06 2024-12-11T20:36:20,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/b16d7c754e4f49668899740ae0fa5723 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b16d7c754e4f49668899740ae0fa5723 2024-12-11T20:36:20,220 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b16d7c754e4f49668899740ae0fa5723, entries=150, sequenceid=48, filesize=11.7 K 2024-12-11T20:36:20,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/8804f8fb42da40ee86a62cfb59abf18e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/8804f8fb42da40ee86a62cfb59abf18e 2024-12-11T20:36:20,224 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/8804f8fb42da40ee86a62cfb59abf18e, entries=150, sequenceid=48, filesize=11.7 K 2024-12-11T20:36:20,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/3aa57af9592c466197db8c52e4b71c06 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3aa57af9592c466197db8c52e4b71c06 2024-12-11T20:36:20,227 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3aa57af9592c466197db8c52e4b71c06, entries=150, sequenceid=48, filesize=11.7 K 2024-12-11T20:36:20,228 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for da1a8ec404804dae6a302e6015dbd206 in 1243ms, sequenceid=48, compaction requested=false 2024-12-11T20:36:20,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:20,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:20,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-11T20:36:20,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-11T20:36:20,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-11T20:36:20,231 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8630 sec 2024-12-11T20:36:20,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8700 sec 2024-12-11T20:36:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:20,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T20:36:20,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:20,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:20,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:20,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:20,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:20,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:20,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/046f7b31152c4c9b93925f9d02745e98 is 50, key is test_row_0/A:col10/1733949380249/Put/seqid=0 2024-12-11T20:36:20,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742284_1460 (size=12001) 2024-12-11T20:36:20,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949440390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:20,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949440507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:20,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-11T20:36:20,517 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:20,517 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:20,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-11T20:36:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T20:36:20,519 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:20,519 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:20,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T20:36:20,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41224 deadline: 1733949440637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:20,646 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:20,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:20,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41246 deadline: 1733949440647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:20,655 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:20,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41222 deadline: 1733949440656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:20,658 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:20,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/046f7b31152c4c9b93925f9d02745e98 2024-12-11T20:36:20,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/de8c672096e641e2bc79a5e4ec8ee1aa is 50, key is test_row_0/B:col10/1733949380249/Put/seqid=0 2024-12-11T20:36:20,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742285_1461 (size=12001) 2024-12-11T20:36:20,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:20,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:20,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:20,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:20,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:20,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:20,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1733949440666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:20,675 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:20,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:20,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949440717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:20,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:20,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:20,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:20,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:20,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:20,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:20,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:20,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T20:36:20,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:20,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:20,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:20,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:20,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:20,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:21,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949441027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:21,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:21,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/de8c672096e641e2bc79a5e4ec8ee1aa 2024-12-11T20:36:21,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/fa647d3c351c439ebf14185be248ed70 is 50, key is test_row_0/C:col10/1733949380249/Put/seqid=0 2024-12-11T20:36:21,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742286_1462 (size=12001) 2024-12-11T20:36:21,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:21,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:21,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T20:36:21,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:21,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:21,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:21,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:21,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/fa647d3c351c439ebf14185be248ed70 2024-12-11T20:36:21,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/046f7b31152c4c9b93925f9d02745e98 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/046f7b31152c4c9b93925f9d02745e98 2024-12-11T20:36:21,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/046f7b31152c4c9b93925f9d02745e98, entries=150, sequenceid=60, filesize=11.7 K 2024-12-11T20:36:21,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/de8c672096e641e2bc79a5e4ec8ee1aa as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/de8c672096e641e2bc79a5e4ec8ee1aa 2024-12-11T20:36:21,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/de8c672096e641e2bc79a5e4ec8ee1aa, entries=150, sequenceid=60, filesize=11.7 K 2024-12-11T20:36:21,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/fa647d3c351c439ebf14185be248ed70 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/fa647d3c351c439ebf14185be248ed70 2024-12-11T20:36:21,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/fa647d3c351c439ebf14185be248ed70, entries=150, sequenceid=60, filesize=11.7 K 2024-12-11T20:36:21,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for da1a8ec404804dae6a302e6015dbd206 in 1245ms, sequenceid=60, compaction requested=true 2024-12-11T20:36:21,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:21,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:21,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:21,496 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:21,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:21,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:21,496 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:21,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:21,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:21,496 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:21,496 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/B is initiating minor compaction (all files) 2024-12-11T20:36:21,496 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:21,496 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/B in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,496 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/A is initiating minor compaction (all files) 2024-12-11T20:36:21,496 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/A in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,496 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/78ff95b7401a4d97b4c4cf2039f09114, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/8804f8fb42da40ee86a62cfb59abf18e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/de8c672096e641e2bc79a5e4ec8ee1aa] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=35.2 K 2024-12-11T20:36:21,497 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/9206b011e68e46569f9d99cd8ac4013d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b16d7c754e4f49668899740ae0fa5723, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/046f7b31152c4c9b93925f9d02745e98] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=37.4 K 2024-12-11T20:36:21,497 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 78ff95b7401a4d97b4c4cf2039f09114, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733949378372 2024-12-11T20:36:21,497 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9206b011e68e46569f9d99cd8ac4013d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733949378372 2024-12-11T20:36:21,497 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b16d7c754e4f49668899740ae0fa5723, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733949378479 2024-12-11T20:36:21,497 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 8804f8fb42da40ee86a62cfb59abf18e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733949378479 2024-12-11T20:36:21,497 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 046f7b31152c4c9b93925f9d02745e98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733949379089 2024-12-11T20:36:21,497 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting de8c672096e641e2bc79a5e4ec8ee1aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733949379089 2024-12-11T20:36:21,504 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#B#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:21,505 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/535252bf251f49848d34114da7ccbc76 is 50, key is test_row_0/B:col10/1733949380249/Put/seqid=0 2024-12-11T20:36:21,505 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:21,505 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/9e04b72d853841cdb514a8adaefeb547 is 50, key is test_row_0/A:col10/1733949380249/Put/seqid=0 2024-12-11T20:36:21,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742288_1464 (size=12104) 2024-12-11T20:36:21,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742287_1463 (size=12104) 2024-12-11T20:36:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:21,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-11T20:36:21,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:21,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949380314, max timestamp=1733949381542Num uniques 859; 2024-12-11T20:36:21,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:21,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949380314, max timestamp=1733949381542Num uniques 859; 2024-12-11T20:36:21,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949380314, max timestamp=1733949381542Num uniques 859; 2024-12-11T20:36:21,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:21,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:21,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/f10c00be8e284989812fa507419597ec is 50, key is test_row_0/A:col10/1733949380387/Put/seqid=0 2024-12-11T20:36:21,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742289_1465 (size=14341) 2024-12-11T20:36:21,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:21,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:21,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T20:36:21,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949381639, max timestamp=1733949381676Num uniques 859; 2024-12-11T20:36:21,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949381639, max timestamp=1733949381676Num uniques 859; 2024-12-11T20:36:21,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:21,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:21,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949381639, max timestamp=1733949381676Num uniques 859; 2024-12-11T20:36:21,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:21,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:21,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:21,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949381770, max timestamp=1733949381797Num uniques 1018; 2024-12-11T20:36:21,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T20:36:21,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949381770, max timestamp=1733949381797Num uniques 1018; 2024-12-11T20:36:21,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949381770, max timestamp=1733949381797Num uniques 1018; 2024-12-11T20:36:21,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T20:36:21,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T20:36:21,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:21,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:21,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:21,918 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/9e04b72d853841cdb514a8adaefeb547 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/9e04b72d853841cdb514a8adaefeb547 2024-12-11T20:36:21,918 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/535252bf251f49848d34114da7ccbc76 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/535252bf251f49848d34114da7ccbc76 2024-12-11T20:36:21,922 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/B of da1a8ec404804dae6a302e6015dbd206 into 535252bf251f49848d34114da7ccbc76(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:21,922 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/A of da1a8ec404804dae6a302e6015dbd206 into 9e04b72d853841cdb514a8adaefeb547(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:21,922 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:21,922 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:21,922 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/A, priority=13, startTime=1733949381496; duration=0sec 2024-12-11T20:36:21,922 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/B, priority=13, startTime=1733949381496; duration=0sec 2024-12-11T20:36:21,922 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:21,922 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:21,922 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:A 2024-12-11T20:36:21,922 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:B 2024-12-11T20:36:21,922 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:21,923 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:21,923 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/C is initiating minor compaction (all files) 2024-12-11T20:36:21,923 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/C in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:21,923 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/45e62147eada41a6b6094d59f8a72bea, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3aa57af9592c466197db8c52e4b71c06, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/fa647d3c351c439ebf14185be248ed70] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=35.2 K 2024-12-11T20:36:21,924 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45e62147eada41a6b6094d59f8a72bea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733949378372 2024-12-11T20:36:21,924 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa57af9592c466197db8c52e4b71c06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733949378479 2024-12-11T20:36:21,924 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa647d3c351c439ebf14185be248ed70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733949379089 2024-12-11T20:36:21,929 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:21,930 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/9bbac62f7ed04c4fbcb7a9f1ebd0a90d is 50, key is test_row_0/C:col10/1733949380249/Put/seqid=0 2024-12-11T20:36:21,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742290_1466 (size=12104) 2024-12-11T20:36:21,936 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/9bbac62f7ed04c4fbcb7a9f1ebd0a90d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/9bbac62f7ed04c4fbcb7a9f1ebd0a90d 2024-12-11T20:36:21,941 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/C of da1a8ec404804dae6a302e6015dbd206 into 9bbac62f7ed04c4fbcb7a9f1ebd0a90d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:21,941 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:21,941 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/C, priority=13, startTime=1733949381496; duration=0sec 2024-12-11T20:36:21,941 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:21,941 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:C 2024-12-11T20:36:21,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733949381904, max timestamp=1733949381930Num uniques 1027; 2024-12-11T20:36:21,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T20:36:21,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733949381904, max timestamp=1733949381930Num uniques 1027; 2024-12-11T20:36:21,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733949381904, max timestamp=1733949381930Num uniques 1027; 2024-12-11T20:36:21,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T20:36:21,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T20:36:21,952 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/f10c00be8e284989812fa507419597ec 2024-12-11T20:36:21,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/c95d84b0c2264a14b08f018386344b93 is 50, key is test_row_0/B:col10/1733949380387/Put/seqid=0 2024-12-11T20:36:21,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742291_1467 (size=12001) 2024-12-11T20:36:21,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/c95d84b0c2264a14b08f018386344b93 2024-12-11T20:36:21,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/ee0ec6e561d04032b0edf363dbfb94e3 is 50, key is test_row_0/C:col10/1733949380387/Put/seqid=0 2024-12-11T20:36:21,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742292_1468 (size=12001) 2024-12-11T20:36:21,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/ee0ec6e561d04032b0edf363dbfb94e3 2024-12-11T20:36:21,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/f10c00be8e284989812fa507419597ec as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f10c00be8e284989812fa507419597ec 2024-12-11T20:36:21,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f10c00be8e284989812fa507419597ec, entries=200, sequenceid=81, filesize=14.0 K 2024-12-11T20:36:21,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/c95d84b0c2264a14b08f018386344b93 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c95d84b0c2264a14b08f018386344b93 2024-12-11T20:36:21,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c95d84b0c2264a14b08f018386344b93, entries=150, sequenceid=81, filesize=11.7 K 2024-12-11T20:36:21,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/ee0ec6e561d04032b0edf363dbfb94e3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/ee0ec6e561d04032b0edf363dbfb94e3 2024-12-11T20:36:21,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/ee0ec6e561d04032b0edf363dbfb94e3, entries=150, sequenceid=81, filesize=11.7 K 2024-12-11T20:36:21,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=60.38 KB/61830 for da1a8ec404804dae6a302e6015dbd206 in 443ms, sequenceid=81, compaction requested=false 2024-12-11T20:36:21,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:21,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=67.09 KB heapSize=184.08 KB 2024-12-11T20:36:21,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:21,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:21,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:21,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:21,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:21,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:22,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/18ae0a9391f548c8a63b05289c7dbd5b is 50, key is test_row_0/A:col10/1733949381982/Put/seqid=0 2024-12-11T20:36:22,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742293_1469 (size=14541) 2024-12-11T20:36:22,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/18ae0a9391f548c8a63b05289c7dbd5b 2024-12-11T20:36:22,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/b81cd0c295084e3d82181141ce1fff21 is 50, key is test_row_0/B:col10/1733949381982/Put/seqid=0 2024-12-11T20:36:22,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742294_1470 (size=12151) 2024-12-11T20:36:22,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/b81cd0c295084e3d82181141ce1fff21 2024-12-11T20:36:22,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:22,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/e3685f4c906f4dd580737bf455a050c2 is 50, key is test_row_0/C:col10/1733949381982/Put/seqid=0 2024-12-11T20:36:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:22,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:22,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742295_1471 (size=12151) 2024-12-11T20:36:22,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949442184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:22,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:22,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:22,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:22,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949442306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:22,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:22,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:22,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:22,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:22,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:22,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:22,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/e3685f4c906f4dd580737bf455a050c2 2024-12-11T20:36:22,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/18ae0a9391f548c8a63b05289c7dbd5b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/18ae0a9391f548c8a63b05289c7dbd5b 2024-12-11T20:36:22,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/18ae0a9391f548c8a63b05289c7dbd5b, entries=200, sequenceid=145, filesize=14.2 K 2024-12-11T20:36:22,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/b81cd0c295084e3d82181141ce1fff21 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/b81cd0c295084e3d82181141ce1fff21 2024-12-11T20:36:22,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/b81cd0c295084e3d82181141ce1fff21, entries=150, sequenceid=145, filesize=11.9 K 2024-12-11T20:36:22,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/e3685f4c906f4dd580737bf455a050c2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/e3685f4c906f4dd580737bf455a050c2 2024-12-11T20:36:22,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/e3685f4c906f4dd580737bf455a050c2, entries=150, sequenceid=145, filesize=11.9 K 2024-12-11T20:36:22,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~184.03 KB/188448, currentSize=114.05 KB/116790 for da1a8ec404804dae6a302e6015dbd206 in 488ms, sequenceid=145, compaction requested=true 2024-12-11T20:36:22,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:22,483 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:22,483 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:22,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:22,483 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:22,483 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40986 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:22,483 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/A is initiating minor compaction (all files) 2024-12-11T20:36:22,483 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/B is initiating minor compaction (all files) 2024-12-11T20:36:22,484 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/B in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:22,484 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/A in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:22,484 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/9e04b72d853841cdb514a8adaefeb547, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f10c00be8e284989812fa507419597ec, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/18ae0a9391f548c8a63b05289c7dbd5b] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=40.0 K 2024-12-11T20:36:22,484 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/535252bf251f49848d34114da7ccbc76, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c95d84b0c2264a14b08f018386344b93, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/b81cd0c295084e3d82181141ce1fff21] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=35.4 K 2024-12-11T20:36:22,484 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 535252bf251f49848d34114da7ccbc76, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733949379089 2024-12-11T20:36:22,484 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e04b72d853841cdb514a8adaefeb547, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733949379089 2024-12-11T20:36:22,484 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting f10c00be8e284989812fa507419597ec, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733949380314 2024-12-11T20:36:22,485 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c95d84b0c2264a14b08f018386344b93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733949380314 2024-12-11T20:36:22,485 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18ae0a9391f548c8a63b05289c7dbd5b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733949381926 2024-12-11T20:36:22,485 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting b81cd0c295084e3d82181141ce1fff21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733949381974 2024-12-11T20:36:22,496 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#A#compaction#395 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:22,496 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/b504f279d82647f7bf351263c978108e is 50, key is test_row_0/A:col10/1733949381982/Put/seqid=0 2024-12-11T20:36:22,499 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#B#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:22,499 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/85071e00d3774c6480af66e7aae45441 is 50, key is test_row_0/B:col10/1733949381982/Put/seqid=0 2024-12-11T20:36:22,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742296_1472 (size=12357) 2024-12-11T20:36:22,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742297_1473 (size=12357) 2024-12-11T20:36:22,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-11T20:36:22,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:22,516 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T20:36:22,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:22,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:22,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:22,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:22,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:22,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:22,518 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/85071e00d3774c6480af66e7aae45441 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/85071e00d3774c6480af66e7aae45441 2024-12-11T20:36:22,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/7aeedb6c51de446c97fb993809f40ab8 is 50, key is test_row_0/A:col10/1733949382183/Put/seqid=0 2024-12-11T20:36:22,525 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/B of da1a8ec404804dae6a302e6015dbd206 into 85071e00d3774c6480af66e7aae45441(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:22,525 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:22,525 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/B, priority=13, startTime=1733949382483; duration=0sec 2024-12-11T20:36:22,526 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:22,526 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:B 2024-12-11T20:36:22,526 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:22,532 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:22,532 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/C is initiating minor compaction (all files) 2024-12-11T20:36:22,532 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/C in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:22,533 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/9bbac62f7ed04c4fbcb7a9f1ebd0a90d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/ee0ec6e561d04032b0edf363dbfb94e3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/e3685f4c906f4dd580737bf455a050c2] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=35.4 K 2024-12-11T20:36:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:22,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:22,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742298_1474 (size=12151) 2024-12-11T20:36:22,535 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bbac62f7ed04c4fbcb7a9f1ebd0a90d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733949379089 2024-12-11T20:36:22,535 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting ee0ec6e561d04032b0edf363dbfb94e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733949380314 2024-12-11T20:36:22,536 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting e3685f4c906f4dd580737bf455a050c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733949381974 2024-12-11T20:36:22,541 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:22,541 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/887f40b7af7e4d27b13ca189aaf44d86 is 50, key is test_row_0/C:col10/1733949381982/Put/seqid=0 2024-12-11T20:36:22,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742299_1475 (size=12357) 2024-12-11T20:36:22,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949442592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:22,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:22,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41224 deadline: 1733949442656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:22,662 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:22,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T20:36:22,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:22,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41222 deadline: 1733949442666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:22,670 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:22,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:22,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41246 deadline: 1733949442677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:22,683 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:22,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,684 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:22,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1733949442697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:22,700 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:22,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:22,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949442706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:22,906 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/b504f279d82647f7bf351263c978108e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b504f279d82647f7bf351263c978108e 2024-12-11T20:36:22,909 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/A of da1a8ec404804dae6a302e6015dbd206 into b504f279d82647f7bf351263c978108e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:22,909 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:22,909 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/A, priority=13, startTime=1733949382483; duration=0sec 2024-12-11T20:36:22,909 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:22,909 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:A 2024-12-11T20:36:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949442917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:22,936 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/7aeedb6c51de446c97fb993809f40ab8 2024-12-11T20:36:22,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/d13f68c513a4400b9f0aa1fbf972b2a4 is 50, key is test_row_0/B:col10/1733949382183/Put/seqid=0 2024-12-11T20:36:22,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742300_1476 (size=12151) 2024-12-11T20:36:22,955 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/887f40b7af7e4d27b13ca189aaf44d86 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/887f40b7af7e4d27b13ca189aaf44d86 2024-12-11T20:36:22,958 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/C of da1a8ec404804dae6a302e6015dbd206 into 887f40b7af7e4d27b13ca189aaf44d86(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:22,958 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:22,958 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/C, priority=13, startTime=1733949382483; duration=0sec 2024-12-11T20:36:22,958 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:22,958 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:C 2024-12-11T20:36:23,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949443226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:23,231 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:23,348 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/d13f68c513a4400b9f0aa1fbf972b2a4 2024-12-11T20:36:23,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/3e278444d9974c9fb4db1de20781314c is 50, key is test_row_0/C:col10/1733949382183/Put/seqid=0 2024-12-11T20:36:23,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742301_1477 (size=12151) 2024-12-11T20:36:23,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949443736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:23,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:23,758 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/3e278444d9974c9fb4db1de20781314c 2024-12-11T20:36:23,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/7aeedb6c51de446c97fb993809f40ab8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/7aeedb6c51de446c97fb993809f40ab8 2024-12-11T20:36:23,765 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/7aeedb6c51de446c97fb993809f40ab8, entries=150, sequenceid=165, filesize=11.9 K 2024-12-11T20:36:23,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/d13f68c513a4400b9f0aa1fbf972b2a4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/d13f68c513a4400b9f0aa1fbf972b2a4 2024-12-11T20:36:23,769 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/d13f68c513a4400b9f0aa1fbf972b2a4, entries=150, sequenceid=165, filesize=11.9 K 2024-12-11T20:36:23,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/3e278444d9974c9fb4db1de20781314c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3e278444d9974c9fb4db1de20781314c 2024-12-11T20:36:23,772 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3e278444d9974c9fb4db1de20781314c, entries=150, sequenceid=165, filesize=11.9 K 2024-12-11T20:36:23,773 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for da1a8ec404804dae6a302e6015dbd206 in 1258ms, sequenceid=165, compaction requested=false 2024-12-11T20:36:23,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:23,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:23,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-11T20:36:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-11T20:36:23,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-11T20:36:23,775 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2550 sec 2024-12-11T20:36:23,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 3.2590 sec 2024-12-11T20:36:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-11T20:36:24,687 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:24,687 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-11T20:36:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T20:36:24,689 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:24,690 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:24,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:24,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T20:36:24,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:24,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:24,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:24,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:24,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:24,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:24,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/bc4b6f62cba043d29aedc227da574212 is 50, key is test_row_0/A:col10/1733949382547/Put/seqid=0 2024-12-11T20:36:24,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742302_1478 (size=14541) 2024-12-11T20:36:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T20:36:24,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:24,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:24,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:24,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:24,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:24,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:24,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949444859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:24,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949444967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:24,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:24,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:24,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:24,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:24,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:24,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:24,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T20:36:25,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/bc4b6f62cba043d29aedc227da574212 2024-12-11T20:36:25,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/ff02feeea9f24d3da40390e82c6e0f0e is 50, key is test_row_0/B:col10/1733949382547/Put/seqid=0 2024-12-11T20:36:25,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742303_1479 (size=12151) 2024-12-11T20:36:25,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949445177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:25,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:25,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:25,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T20:36:25,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:25,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:25,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949445486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:25,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/ff02feeea9f24d3da40390e82c6e0f0e 2024-12-11T20:36:25,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/5fd85d4c109748e58a2c8f49f401fe1b is 50, key is test_row_0/C:col10/1733949382547/Put/seqid=0 2024-12-11T20:36:25,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742304_1480 (size=12151) 2024-12-11T20:36:25,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:25,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:25,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:25,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:25,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T20:36:25,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:25,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:25,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:25,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:25,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=181 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/5fd85d4c109748e58a2c8f49f401fe1b 2024-12-11T20:36:25,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/bc4b6f62cba043d29aedc227da574212 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/bc4b6f62cba043d29aedc227da574212 2024-12-11T20:36:25,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/bc4b6f62cba043d29aedc227da574212, entries=200, sequenceid=181, filesize=14.2 K 2024-12-11T20:36:25,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/ff02feeea9f24d3da40390e82c6e0f0e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ff02feeea9f24d3da40390e82c6e0f0e 2024-12-11T20:36:26,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ff02feeea9f24d3da40390e82c6e0f0e, entries=150, sequenceid=181, filesize=11.9 K 2024-12-11T20:36:26,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/5fd85d4c109748e58a2c8f49f401fe1b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/5fd85d4c109748e58a2c8f49f401fe1b 2024-12-11T20:36:26,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/5fd85d4c109748e58a2c8f49f401fe1b, entries=150, sequenceid=181, filesize=11.9 K 2024-12-11T20:36:26,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for da1a8ec404804dae6a302e6015dbd206 in 1252ms, sequenceid=181, compaction requested=true 2024-12-11T20:36:26,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:26,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:26,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:26,005 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:26,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:26,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:26,005 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:26,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:26,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:26,006 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:26,006 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:26,007 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/A is initiating minor compaction (all files) 2024-12-11T20:36:26,007 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/B is initiating minor compaction (all files) 2024-12-11T20:36:26,007 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/A in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,007 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/B in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,007 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/85071e00d3774c6480af66e7aae45441, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/d13f68c513a4400b9f0aa1fbf972b2a4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ff02feeea9f24d3da40390e82c6e0f0e] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=35.8 K 2024-12-11T20:36:26,007 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b504f279d82647f7bf351263c978108e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/7aeedb6c51de446c97fb993809f40ab8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/bc4b6f62cba043d29aedc227da574212] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=38.1 K 2024-12-11T20:36:26,007 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b504f279d82647f7bf351263c978108e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733949381974 2024-12-11T20:36:26,007 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 85071e00d3774c6480af66e7aae45441, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733949381974 2024-12-11T20:36:26,007 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aeedb6c51de446c97fb993809f40ab8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733949382143 2024-12-11T20:36:26,007 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d13f68c513a4400b9f0aa1fbf972b2a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733949382143 2024-12-11T20:36:26,007 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc4b6f62cba043d29aedc227da574212, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733949382547 2024-12-11T20:36:26,007 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting ff02feeea9f24d3da40390e82c6e0f0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733949382547 2024-12-11T20:36:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:26,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T20:36:26,013 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#B#compaction#405 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:26,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:26,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:26,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:26,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:26,013 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#A#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:26,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:26,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:26,013 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/467f8d4b8066466e9fe3a12b8be2a7a8 is 50, key is test_row_0/B:col10/1733949382547/Put/seqid=0 2024-12-11T20:36:26,013 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/896b12cd0bfc48749224aa0f2d618ce0 is 50, key is test_row_0/A:col10/1733949382547/Put/seqid=0 2024-12-11T20:36:26,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/4b207ab6a7d0447fb2d377e6b270efa8 is 50, key is test_row_0/A:col10/1733949386011/Put/seqid=0 2024-12-11T20:36:26,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742305_1481 (size=12459) 2024-12-11T20:36:26,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742306_1482 (size=12459) 2024-12-11T20:36:26,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742307_1483 (size=14541) 2024-12-11T20:36:26,024 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/467f8d4b8066466e9fe3a12b8be2a7a8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/467f8d4b8066466e9fe3a12b8be2a7a8 2024-12-11T20:36:26,026 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/896b12cd0bfc48749224aa0f2d618ce0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/896b12cd0bfc48749224aa0f2d618ce0 2024-12-11T20:36:26,029 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/B of da1a8ec404804dae6a302e6015dbd206 into 467f8d4b8066466e9fe3a12b8be2a7a8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:26,029 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:26,029 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/B, priority=13, startTime=1733949386005; duration=0sec 2024-12-11T20:36:26,029 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:26,029 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:B 2024-12-11T20:36:26,029 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:26,030 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/A of da1a8ec404804dae6a302e6015dbd206 into 896b12cd0bfc48749224aa0f2d618ce0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:26,030 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:26,030 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/A, priority=13, startTime=1733949386005; duration=0sec 2024-12-11T20:36:26,030 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:26,030 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:A 2024-12-11T20:36:26,030 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:26,030 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/C is initiating minor compaction (all files) 2024-12-11T20:36:26,030 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/C in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,030 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/887f40b7af7e4d27b13ca189aaf44d86, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3e278444d9974c9fb4db1de20781314c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/5fd85d4c109748e58a2c8f49f401fe1b] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=35.8 K 2024-12-11T20:36:26,030 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 887f40b7af7e4d27b13ca189aaf44d86, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733949381974 2024-12-11T20:36:26,031 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e278444d9974c9fb4db1de20781314c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1733949382143 2024-12-11T20:36:26,031 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fd85d4c109748e58a2c8f49f401fe1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733949382547 2024-12-11T20:36:26,038 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:26,038 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/062ee011c75b47c8b3a7b90f58bc4f9e is 50, key is test_row_0/C:col10/1733949382547/Put/seqid=0 2024-12-11T20:36:26,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742308_1484 (size=12459) 2024-12-11T20:36:26,044 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/062ee011c75b47c8b3a7b90f58bc4f9e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/062ee011c75b47c8b3a7b90f58bc4f9e 2024-12-11T20:36:26,048 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/C of da1a8ec404804dae6a302e6015dbd206 into 062ee011c75b47c8b3a7b90f58bc4f9e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:26,048 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:26,048 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/C, priority=13, startTime=1733949386005; duration=0sec 2024-12-11T20:36:26,048 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:26,048 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:C 2024-12-11T20:36:26,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:26,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:26,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949446066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:26,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949446176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:26,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:26,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:26,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949446387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:26,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/4b207ab6a7d0447fb2d377e6b270efa8 2024-12-11T20:36:26,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/e486fb57822241f5ad76999a06118408 is 50, key is test_row_0/B:col10/1733949386011/Put/seqid=0 2024-12-11T20:36:26,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742309_1485 (size=12151) 2024-12-11T20:36:26,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:26,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:26,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:26,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:26,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:26,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41222 deadline: 1733949446676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:26,679 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:26,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:26,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41246 deadline: 1733949446697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:26,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949446697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:26,698 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:26,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41224 deadline: 1733949446707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:26,712 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:26,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:26,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1733949446727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:26,732 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:26,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:26,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:26,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T20:36:26,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/e486fb57822241f5ad76999a06118408 2024-12-11T20:36:26,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/f5eb63d8f03d4911934f68f5d69b39d9 is 50, key is test_row_0/C:col10/1733949386011/Put/seqid=0 2024-12-11T20:36:26,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742310_1486 (size=12151) 2024-12-11T20:36:26,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:26,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:27,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:27,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:27,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:27,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:27,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:27,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:27,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949447207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:27,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:27,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:27,210 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:27,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/f5eb63d8f03d4911934f68f5d69b39d9 2024-12-11T20:36:27,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/4b207ab6a7d0447fb2d377e6b270efa8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/4b207ab6a7d0447fb2d377e6b270efa8 2024-12-11T20:36:27,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/4b207ab6a7d0447fb2d377e6b270efa8, entries=200, sequenceid=201, filesize=14.2 K 2024-12-11T20:36:27,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/e486fb57822241f5ad76999a06118408 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/e486fb57822241f5ad76999a06118408 2024-12-11T20:36:27,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/e486fb57822241f5ad76999a06118408, entries=150, sequenceid=201, filesize=11.9 K 2024-12-11T20:36:27,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/f5eb63d8f03d4911934f68f5d69b39d9 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/f5eb63d8f03d4911934f68f5d69b39d9 2024-12-11T20:36:27,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/f5eb63d8f03d4911934f68f5d69b39d9, entries=150, sequenceid=201, filesize=11.9 K 2024-12-11T20:36:27,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for da1a8ec404804dae6a302e6015dbd206 in 1262ms, sequenceid=201, compaction requested=false 2024-12-11T20:36:27,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:27,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-11T20:36:27,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:27,287 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T20:36:27,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:27,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:27,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:27,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:27,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:27,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:27,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/b93e8d386f5e474a91dbfd58930a7577 is 50, key is test_row_0/A:col10/1733949386064/Put/seqid=0 2024-12-11T20:36:27,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742311_1487 (size=12151) 2024-12-11T20:36:27,695 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/b93e8d386f5e474a91dbfd58930a7577 2024-12-11T20:36:27,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/c6bd142c3066472eb1a7d83ec5380728 is 50, key is test_row_0/B:col10/1733949386064/Put/seqid=0 2024-12-11T20:36:27,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742312_1488 (size=12151) 2024-12-11T20:36:27,749 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/c6bd142c3066472eb1a7d83ec5380728 2024-12-11T20:36:27,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/059a4784958744b3ac05047385ab94de is 50, key is test_row_0/C:col10/1733949386064/Put/seqid=0 2024-12-11T20:36:27,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742313_1489 (size=12151) 2024-12-11T20:36:28,160 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/059a4784958744b3ac05047385ab94de 2024-12-11T20:36:28,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/b93e8d386f5e474a91dbfd58930a7577 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b93e8d386f5e474a91dbfd58930a7577 2024-12-11T20:36:28,166 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b93e8d386f5e474a91dbfd58930a7577, entries=150, sequenceid=216, filesize=11.9 K 2024-12-11T20:36:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/c6bd142c3066472eb1a7d83ec5380728 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c6bd142c3066472eb1a7d83ec5380728 2024-12-11T20:36:28,169 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c6bd142c3066472eb1a7d83ec5380728, entries=150, sequenceid=216, filesize=11.9 K 2024-12-11T20:36:28,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/059a4784958744b3ac05047385ab94de as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/059a4784958744b3ac05047385ab94de 2024-12-11T20:36:28,173 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/059a4784958744b3ac05047385ab94de, entries=150, sequenceid=216, filesize=11.9 K 2024-12-11T20:36:28,173 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for da1a8ec404804dae6a302e6015dbd206 in 886ms, sequenceid=216, compaction requested=true 2024-12-11T20:36:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-11T20:36:28,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-11T20:36:28,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-11T20:36:28,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4850 sec 2024-12-11T20:36:28,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 3.4890 sec 2024-12-11T20:36:28,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:28,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-11T20:36:28,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:28,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:28,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:28,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:28,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:28,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:28,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/1d42c4baab8b4c1aa5334c6644254668 is 50, key is test_row_0/A:col10/1733949388222/Put/seqid=0 2024-12-11T20:36:28,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742314_1490 (size=14541) 2024-12-11T20:36:28,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949388325, max timestamp=1733949388350Num uniques 859; 2024-12-11T20:36:28,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:28,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949388325, max timestamp=1733949388350Num uniques 859; 2024-12-11T20:36:28,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949388325, max timestamp=1733949388350Num uniques 859; 2024-12-11T20:36:28,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:28,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949448443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:28,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:28,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:28,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949448557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:28,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:28,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/1d42c4baab8b4c1aa5334c6644254668 2024-12-11T20:36:28,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/ae832cc635064df09db9b381effe3e4d is 50, key is test_row_0/B:col10/1733949388222/Put/seqid=0 2024-12-11T20:36:28,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742315_1491 (size=12151) 2024-12-11T20:36:28,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949448767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:28,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-11T20:36:28,857 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:28,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-11T20:36:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T20:36:28,859 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:28,859 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:28,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T20:36:29,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T20:36:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/ae832cc635064df09db9b381effe3e4d 2024-12-11T20:36:29,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/c6b04d443581407f8d9321a208988e92 is 50, key is test_row_0/C:col10/1733949388222/Put/seqid=0 2024-12-11T20:36:29,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742316_1492 (size=12151) 2024-12-11T20:36:29,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949449076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:29,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T20:36:29,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:29,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T20:36:29,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T20:36:29,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:29,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/c6b04d443581407f8d9321a208988e92 2024-12-11T20:36:29,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T20:36:29,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:29,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/1d42c4baab8b4c1aa5334c6644254668 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/1d42c4baab8b4c1aa5334c6644254668 2024-12-11T20:36:29,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/1d42c4baab8b4c1aa5334c6644254668, entries=200, sequenceid=226, filesize=14.2 K 2024-12-11T20:36:29,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/ae832cc635064df09db9b381effe3e4d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ae832cc635064df09db9b381effe3e4d 2024-12-11T20:36:29,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ae832cc635064df09db9b381effe3e4d, entries=150, sequenceid=226, filesize=11.9 K 2024-12-11T20:36:29,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/c6b04d443581407f8d9321a208988e92 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c6b04d443581407f8d9321a208988e92 2024-12-11T20:36:29,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c6b04d443581407f8d9321a208988e92, entries=150, sequenceid=226, filesize=11.9 K 2024-12-11T20:36:29,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for da1a8ec404804dae6a302e6015dbd206 in 1238ms, sequenceid=226, compaction requested=true 2024-12-11T20:36:29,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:29,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:29,481 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:36:29,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:29,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:29,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:29,481 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:36:29,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:29,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:29,482 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53692 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:36:29,482 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/A is initiating minor compaction (all files) 2024-12-11T20:36:29,483 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:36:29,483 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/A in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,483 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/B is initiating minor compaction (all files) 2024-12-11T20:36:29,483 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/896b12cd0bfc48749224aa0f2d618ce0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/4b207ab6a7d0447fb2d377e6b270efa8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b93e8d386f5e474a91dbfd58930a7577, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/1d42c4baab8b4c1aa5334c6644254668] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=52.4 K 2024-12-11T20:36:29,483 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/B in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,483 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/467f8d4b8066466e9fe3a12b8be2a7a8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/e486fb57822241f5ad76999a06118408, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c6bd142c3066472eb1a7d83ec5380728, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ae832cc635064df09db9b381effe3e4d] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=47.8 K 2024-12-11T20:36:29,483 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 896b12cd0bfc48749224aa0f2d618ce0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733949382547 2024-12-11T20:36:29,483 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 467f8d4b8066466e9fe3a12b8be2a7a8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733949382547 2024-12-11T20:36:29,483 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b207ab6a7d0447fb2d377e6b270efa8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733949384805 2024-12-11T20:36:29,483 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting e486fb57822241f5ad76999a06118408, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733949384805 2024-12-11T20:36:29,484 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c6bd142c3066472eb1a7d83ec5380728, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733949386037 2024-12-11T20:36:29,484 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b93e8d386f5e474a91dbfd58930a7577, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733949386037 2024-12-11T20:36:29,484 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting ae832cc635064df09db9b381effe3e4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733949388222 2024-12-11T20:36:29,484 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d42c4baab8b4c1aa5334c6644254668, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733949388222 2024-12-11T20:36:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T20:36:29,490 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:29,490 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/70d101a4125a442698ac581cdfdd4e52 is 50, key is test_row_0/B:col10/1733949388222/Put/seqid=0 2024-12-11T20:36:29,490 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:29,491 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/26df2d95aac24b349bb2d62693f7f744 is 50, key is test_row_0/A:col10/1733949388222/Put/seqid=0 2024-12-11T20:36:29,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742317_1493 (size=12595) 2024-12-11T20:36:29,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742318_1494 (size=12595) 2024-12-11T20:36:29,500 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/26df2d95aac24b349bb2d62693f7f744 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/26df2d95aac24b349bb2d62693f7f744 2024-12-11T20:36:29,503 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/A of da1a8ec404804dae6a302e6015dbd206 into 26df2d95aac24b349bb2d62693f7f744(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:29,503 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:29,503 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/A, priority=12, startTime=1733949389481; duration=0sec 2024-12-11T20:36:29,503 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:29,503 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:A 2024-12-11T20:36:29,504 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:36:29,504 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:36:29,504 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/C is initiating minor compaction (all files) 2024-12-11T20:36:29,504 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/C in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,505 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/062ee011c75b47c8b3a7b90f58bc4f9e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/f5eb63d8f03d4911934f68f5d69b39d9, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/059a4784958744b3ac05047385ab94de, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c6b04d443581407f8d9321a208988e92] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=47.8 K 2024-12-11T20:36:29,505 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 062ee011c75b47c8b3a7b90f58bc4f9e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=181, earliestPutTs=1733949382547 2024-12-11T20:36:29,505 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5eb63d8f03d4911934f68f5d69b39d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1733949384805 2024-12-11T20:36:29,505 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 059a4784958744b3ac05047385ab94de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733949386037 2024-12-11T20:36:29,505 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6b04d443581407f8d9321a208988e92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733949388222 2024-12-11T20:36:29,514 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#C#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:29,514 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/c39d513784074034b9d52638504bce6d is 50, key is test_row_0/C:col10/1733949388222/Put/seqid=0 2024-12-11T20:36:29,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742319_1495 (size=12595) 2024-12-11T20:36:29,521 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/c39d513784074034b9d52638504bce6d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c39d513784074034b9d52638504bce6d 2024-12-11T20:36:29,524 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/C of da1a8ec404804dae6a302e6015dbd206 into c39d513784074034b9d52638504bce6d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:29,524 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:29,524 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/C, priority=12, startTime=1733949389481; duration=0sec 2024-12-11T20:36:29,524 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:29,524 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:C 2024-12-11T20:36:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:29,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-12-11T20:36:29,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:29,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T20:36:29,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:29,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T20:36:29,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:29,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-11T20:36:29,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/ef1d98dbf8ac43efb41a28fb1f6a770c is 50, key is test_row_0/A:col10/1733949388423/Put/seqid=0 2024-12-11T20:36:29,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742320_1496 (size=14741) 2024-12-11T20:36:29,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T20:36:29,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:29,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949449628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:29,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:29,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949449736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:29,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:29,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T20:36:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:29,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,898 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/70d101a4125a442698ac581cdfdd4e52 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/70d101a4125a442698ac581cdfdd4e52 2024-12-11T20:36:29,901 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/B of da1a8ec404804dae6a302e6015dbd206 into 70d101a4125a442698ac581cdfdd4e52(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:29,901 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:29,901 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/B, priority=12, startTime=1733949389481; duration=0sec 2024-12-11T20:36:29,901 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:29,901 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:B 2024-12-11T20:36:29,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T20:36:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:29,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:29,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949449947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:29,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T20:36:29,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/ef1d98dbf8ac43efb41a28fb1f6a770c 2024-12-11T20:36:30,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/3edaf1d547cd49209e698479b417e627 is 50, key is test_row_0/B:col10/1733949388423/Put/seqid=0 2024-12-11T20:36:30,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742321_1497 (size=12301) 2024-12-11T20:36:30,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/3edaf1d547cd49209e698479b417e627 2024-12-11T20:36:30,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/62e272bdbe974c8890ee527640cb96e1 is 50, key is test_row_0/C:col10/1733949388423/Put/seqid=0 2024-12-11T20:36:30,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742322_1498 (size=12301) 2024-12-11T20:36:30,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T20:36:30,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:30,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:30,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:30,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:30,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:30,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:30,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T20:36:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:30,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:30,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:30,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:30,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949450266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:30,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T20:36:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:30,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:30,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/62e272bdbe974c8890ee527640cb96e1 2024-12-11T20:36:30,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/ef1d98dbf8ac43efb41a28fb1f6a770c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/ef1d98dbf8ac43efb41a28fb1f6a770c 2024-12-11T20:36:30,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/ef1d98dbf8ac43efb41a28fb1f6a770c, entries=200, sequenceid=266, filesize=14.4 K 2024-12-11T20:36:30,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/3edaf1d547cd49209e698479b417e627 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/3edaf1d547cd49209e698479b417e627 2024-12-11T20:36:30,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/3edaf1d547cd49209e698479b417e627, entries=150, sequenceid=266, filesize=12.0 K 2024-12-11T20:36:30,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/62e272bdbe974c8890ee527640cb96e1 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/62e272bdbe974c8890ee527640cb96e1 2024-12-11T20:36:30,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/62e272bdbe974c8890ee527640cb96e1, entries=150, sequenceid=266, filesize=12.0 K 2024-12-11T20:36:30,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for da1a8ec404804dae6a302e6015dbd206 in 841ms, sequenceid=266, compaction requested=false 2024-12-11T20:36:30,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:30,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-11T20:36:30,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:30,537 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-11T20:36:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:30,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/85a462f8520d4b6f8d6924e2f188e2ba is 50, key is test_row_0/A:col10/1733949389625/Put/seqid=0 2024-12-11T20:36:30,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742323_1499 (size=12301) 2024-12-11T20:36:30,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:30,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:30,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949390836, max timestamp=1733949390869Num uniques 859; 2024-12-11T20:36:30,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:30,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949390836, max timestamp=1733949390869Num uniques 859; 2024-12-11T20:36:30,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:30,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949390836, max timestamp=1733949390869Num uniques 859; 2024-12-11T20:36:30,872 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:30,956 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/85a462f8520d4b6f8d6924e2f188e2ba 2024-12-11T20:36:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/4cdfd690188448bd980adc3238531639 is 50, key is test_row_0/B:col10/1733949389625/Put/seqid=0 2024-12-11T20:36:30,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742324_1500 (size=12301) 2024-12-11T20:36:30,977 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/4cdfd690188448bd980adc3238531639 2024-12-11T20:36:30,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/97caeb35da0b4767beb5a5d3681346c8 is 50, key is test_row_0/C:col10/1733949389625/Put/seqid=0 2024-12-11T20:36:30,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742325_1501 (size=12301) 2024-12-11T20:36:31,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949390942, max timestamp=1733949390999Num uniques 1018; 2024-12-11T20:36:31,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949390942, max timestamp=1733949390999Num uniques 1018; 2024-12-11T20:36:31,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T20:36:31,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T20:36:31,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733949390942, max timestamp=1733949390999Num uniques 1018; 2024-12-11T20:36:31,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-11T20:36:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T20:36:31,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949451117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:31,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733949391089, max timestamp=1733949391115Num uniques 1027; 2024-12-11T20:36:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:31,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T20:36:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:31,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733949391089, max timestamp=1733949391115Num uniques 1027; 2024-12-11T20:36:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:31,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T20:36:31,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733949391089, max timestamp=1733949391115Num uniques 1027; 2024-12-11T20:36:31,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-11T20:36:31,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733949391326, max timestamp=1733949391352Num uniques 1036; 2024-12-11T20:36:31,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-11T20:36:31,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733949391326, max timestamp=1733949391352Num uniques 1036; 2024-12-11T20:36:31,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-11T20:36:31,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733949391326, max timestamp=1733949391352Num uniques 1036; 2024-12-11T20:36:31,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-11T20:36:31,387 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/97caeb35da0b4767beb5a5d3681346c8 2024-12-11T20:36:31,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/85a462f8520d4b6f8d6924e2f188e2ba as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/85a462f8520d4b6f8d6924e2f188e2ba 2024-12-11T20:36:31,392 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/85a462f8520d4b6f8d6924e2f188e2ba, entries=150, sequenceid=276, filesize=12.0 K 2024-12-11T20:36:31,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/4cdfd690188448bd980adc3238531639 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/4cdfd690188448bd980adc3238531639 2024-12-11T20:36:31,396 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/4cdfd690188448bd980adc3238531639, entries=150, sequenceid=276, filesize=12.0 K 2024-12-11T20:36:31,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/97caeb35da0b4767beb5a5d3681346c8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/97caeb35da0b4767beb5a5d3681346c8 2024-12-11T20:36:31,400 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/97caeb35da0b4767beb5a5d3681346c8, entries=150, sequenceid=276, filesize=12.0 K 2024-12-11T20:36:31,401 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=73.80 KB/75570 for da1a8ec404804dae6a302e6015dbd206 in 864ms, sequenceid=276, compaction requested=true 2024-12-11T20:36:31,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:31,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:31,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-11T20:36:31,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-11T20:36:31,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-11T20:36:31,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5430 sec 2024-12-11T20:36:31,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.5470 sec 2024-12-11T20:36:31,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:31,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=80.51 KB heapSize=223.08 KB 2024-12-11T20:36:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:31,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/a0fa7b7a6751478f919a30696e485c13 is 50, key is test_row_0/A:col10/1733949391397/Put/seqid=0 2024-12-11T20:36:31,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742326_1502 (size=14741) 2024-12-11T20:36:31,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 320 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949451521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:31,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:31,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:31,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 321 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949451627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:31,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:31,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/a0fa7b7a6751478f919a30696e485c13 2024-12-11T20:36:31,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/132cf559798b481eb4dfe6da7bbd152e is 50, key is test_row_0/B:col10/1733949391397/Put/seqid=0 2024-12-11T20:36:31,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:31,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 322 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949451836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:31,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:31,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742327_1503 (size=12301) 2024-12-11T20:36:32,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 323 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949452147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:32,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:32,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/132cf559798b481eb4dfe6da7bbd152e 2024-12-11T20:36:32,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/b95e7bff19fb435c99a23e62e0a7687d is 50, key is test_row_0/C:col10/1733949391397/Put/seqid=0 2024-12-11T20:36:32,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742328_1504 (size=12301) 2024-12-11T20:36:32,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/b95e7bff19fb435c99a23e62e0a7687d 2024-12-11T20:36:32,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/a0fa7b7a6751478f919a30696e485c13 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/a0fa7b7a6751478f919a30696e485c13 2024-12-11T20:36:32,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/a0fa7b7a6751478f919a30696e485c13, entries=200, sequenceid=356, filesize=14.4 K 2024-12-11T20:36:32,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/132cf559798b481eb4dfe6da7bbd152e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/132cf559798b481eb4dfe6da7bbd152e 2024-12-11T20:36:32,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949452656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:32,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/132cf559798b481eb4dfe6da7bbd152e, entries=150, sequenceid=356, filesize=12.0 K 2024-12-11T20:36:32,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/b95e7bff19fb435c99a23e62e0a7687d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b95e7bff19fb435c99a23e62e0a7687d 2024-12-11T20:36:32,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b95e7bff19fb435c99a23e62e0a7687d, entries=150, sequenceid=356, filesize=12.0 K 2024-12-11T20:36:32,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~223.03 KB/228384, currentSize=100.63 KB/103050 for da1a8ec404804dae6a302e6015dbd206 in 1261ms, sequenceid=356, compaction requested=true 2024-12-11T20:36:32,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:32,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:32,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:32,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:32,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:32,670 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:36:32,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:32,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:32,670 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:36:32,671 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54378 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:36:32,671 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:36:32,671 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/A is initiating minor compaction (all files) 2024-12-11T20:36:32,671 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/B is initiating minor compaction (all files) 2024-12-11T20:36:32,671 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/A in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:32,671 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/B in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:32,671 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/26df2d95aac24b349bb2d62693f7f744, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/ef1d98dbf8ac43efb41a28fb1f6a770c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/85a462f8520d4b6f8d6924e2f188e2ba, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/a0fa7b7a6751478f919a30696e485c13] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=53.1 K 2024-12-11T20:36:32,671 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/70d101a4125a442698ac581cdfdd4e52, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/3edaf1d547cd49209e698479b417e627, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/4cdfd690188448bd980adc3238531639, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/132cf559798b481eb4dfe6da7bbd152e] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=48.3 K 2024-12-11T20:36:32,672 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 70d101a4125a442698ac581cdfdd4e52, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733949388222 2024-12-11T20:36:32,672 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26df2d95aac24b349bb2d62693f7f744, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733949388222 2024-12-11T20:36:32,673 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef1d98dbf8ac43efb41a28fb1f6a770c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733949388423 2024-12-11T20:36:32,673 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3edaf1d547cd49209e698479b417e627, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733949388423 2024-12-11T20:36:32,673 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cdfd690188448bd980adc3238531639, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733949389616 2024-12-11T20:36:32,673 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85a462f8520d4b6f8d6924e2f188e2ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733949389616 2024-12-11T20:36:32,673 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0fa7b7a6751478f919a30696e485c13, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733949391386 2024-12-11T20:36:32,673 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 132cf559798b481eb4dfe6da7bbd152e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733949391393 2024-12-11T20:36:32,680 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#B#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:32,680 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#A#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:32,680 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/3d4f86904ca74bf1a90590e5bebd7b36 is 50, key is test_row_0/A:col10/1733949391397/Put/seqid=0 2024-12-11T20:36:32,680 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/c553bca5aed84fbbabb833a9ab2edd52 is 50, key is test_row_0/B:col10/1733949391397/Put/seqid=0 2024-12-11T20:36:32,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742330_1506 (size=12881) 2024-12-11T20:36:32,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742329_1505 (size=12881) 2024-12-11T20:36:32,699 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/c553bca5aed84fbbabb833a9ab2edd52 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c553bca5aed84fbbabb833a9ab2edd52 2024-12-11T20:36:32,702 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/B of da1a8ec404804dae6a302e6015dbd206 into c553bca5aed84fbbabb833a9ab2edd52(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:32,702 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:32,702 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/B, priority=12, startTime=1733949392670; duration=0sec 2024-12-11T20:36:32,702 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:32,702 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:B 2024-12-11T20:36:32,702 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:36:32,703 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:36:32,703 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/C is initiating minor compaction (all files) 2024-12-11T20:36:32,703 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/C in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:32,703 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c39d513784074034b9d52638504bce6d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/62e272bdbe974c8890ee527640cb96e1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/97caeb35da0b4767beb5a5d3681346c8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b95e7bff19fb435c99a23e62e0a7687d] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=48.3 K 2024-12-11T20:36:32,703 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c39d513784074034b9d52638504bce6d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1733949388222 2024-12-11T20:36:32,704 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 62e272bdbe974c8890ee527640cb96e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733949388423 2024-12-11T20:36:32,704 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 97caeb35da0b4767beb5a5d3681346c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1733949389616 2024-12-11T20:36:32,704 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting b95e7bff19fb435c99a23e62e0a7687d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733949391393 2024-12-11T20:36:32,710 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:32,710 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/65d65a74cdef43029625ce7c8bd42c1c is 50, key is test_row_0/C:col10/1733949391397/Put/seqid=0 2024-12-11T20:36:32,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742331_1507 (size=12881) 2024-12-11T20:36:32,750 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/65d65a74cdef43029625ce7c8bd42c1c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/65d65a74cdef43029625ce7c8bd42c1c 2024-12-11T20:36:32,753 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/C of da1a8ec404804dae6a302e6015dbd206 into 65d65a74cdef43029625ce7c8bd42c1c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:32,753 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:32,753 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/C, priority=12, startTime=1733949392670; duration=0sec 2024-12-11T20:36:32,753 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:32,753 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:C 2024-12-11T20:36:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-11T20:36:33,037 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:33,038 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-11T20:36:33,039 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:33,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-11T20:36:33,040 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:33,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:33,092 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/3d4f86904ca74bf1a90590e5bebd7b36 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3d4f86904ca74bf1a90590e5bebd7b36 2024-12-11T20:36:33,096 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/A of da1a8ec404804dae6a302e6015dbd206 into 3d4f86904ca74bf1a90590e5bebd7b36(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:33,096 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:33,096 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/A, priority=12, startTime=1733949392670; duration=0sec 2024-12-11T20:36:33,096 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:33,096 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:A 2024-12-11T20:36:33,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-11T20:36:33,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-11T20:36:33,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:33,192 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T20:36:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:33,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:33,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/b8a68b53f73742c0993b7fd95bfa3478 is 50, key is test_row_0/A:col10/1733949391519/Put/seqid=0 2024-12-11T20:36:33,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742332_1508 (size=12301) 2024-12-11T20:36:33,203 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/b8a68b53f73742c0993b7fd95bfa3478 2024-12-11T20:36:33,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/671226a1458449b4904bdb1802cc8438 is 50, key is test_row_0/B:col10/1733949391519/Put/seqid=0 2024-12-11T20:36:33,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742333_1509 (size=12301) 2024-12-11T20:36:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-11T20:36:33,614 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/671226a1458449b4904bdb1802cc8438 2024-12-11T20:36:33,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/b73a8b2b09fe4b899104aaa3e4b2103f is 50, key is test_row_0/C:col10/1733949391519/Put/seqid=0 2024-12-11T20:36:33,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742334_1510 (size=12301) 2024-12-11T20:36:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-11T20:36:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:33,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:33,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949453731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:33,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:33,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 337 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949453847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:33,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:34,023 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/b73a8b2b09fe4b899104aaa3e4b2103f 2024-12-11T20:36:34,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/b8a68b53f73742c0993b7fd95bfa3478 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b8a68b53f73742c0993b7fd95bfa3478 2024-12-11T20:36:34,029 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b8a68b53f73742c0993b7fd95bfa3478, entries=150, sequenceid=377, filesize=12.0 K 2024-12-11T20:36:34,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/671226a1458449b4904bdb1802cc8438 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/671226a1458449b4904bdb1802cc8438 2024-12-11T20:36:34,032 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/671226a1458449b4904bdb1802cc8438, entries=150, sequenceid=377, filesize=12.0 K 2024-12-11T20:36:34,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/b73a8b2b09fe4b899104aaa3e4b2103f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b73a8b2b09fe4b899104aaa3e4b2103f 2024-12-11T20:36:34,036 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b73a8b2b09fe4b899104aaa3e4b2103f, entries=150, sequenceid=377, filesize=12.0 K 2024-12-11T20:36:34,036 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for da1a8ec404804dae6a302e6015dbd206 in 844ms, sequenceid=377, compaction requested=false 2024-12-11T20:36:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-11T20:36:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-11T20:36:34,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-11T20:36:34,039 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 998 msec 2024-12-11T20:36:34,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0020 sec 2024-12-11T20:36:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:34,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T20:36:34,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:34,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:34,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:34,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:34,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:34,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:34,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/f95c2664676f46fd9d78ece21c2265a8 is 50, key is test_row_0/A:col10/1733949394058/Put/seqid=0 2024-12-11T20:36:34,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742335_1511 (size=14741) 2024-12-11T20:36:34,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/f95c2664676f46fd9d78ece21c2265a8 2024-12-11T20:36:34,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/1a0cbebb3e944221813cc84551b67cb3 is 50, key is test_row_0/B:col10/1733949394058/Put/seqid=0 2024-12-11T20:36:34,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742336_1512 (size=12301) 2024-12-11T20:36:34,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949454149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:34,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:34,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-11T20:36:34,177 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:34,178 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:34,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-11T20:36:34,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-11T20:36:34,179 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:34,180 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:34,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:34,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 354 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949454267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:34,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-11T20:36:34,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-11T20:36:34,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:34,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:34,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:34,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:34,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:34,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:34,478 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/1a0cbebb3e944221813cc84551b67cb3 2024-12-11T20:36:34,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 355 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949454476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:34,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-11T20:36:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:34,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:34,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/0bdc598fc2de4f51b6e4def2169f8764 is 50, key is test_row_0/C:col10/1733949394058/Put/seqid=0 2024-12-11T20:36:34,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:34,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:34,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-11T20:36:34,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742337_1513 (size=12301) 2024-12-11T20:36:34,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/0bdc598fc2de4f51b6e4def2169f8764 2024-12-11T20:36:34,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/f95c2664676f46fd9d78ece21c2265a8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f95c2664676f46fd9d78ece21c2265a8 2024-12-11T20:36:34,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f95c2664676f46fd9d78ece21c2265a8, entries=200, sequenceid=392, filesize=14.4 K 2024-12-11T20:36:34,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/1a0cbebb3e944221813cc84551b67cb3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/1a0cbebb3e944221813cc84551b67cb3 2024-12-11T20:36:34,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/1a0cbebb3e944221813cc84551b67cb3, entries=150, sequenceid=392, filesize=12.0 K 2024-12-11T20:36:34,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/0bdc598fc2de4f51b6e4def2169f8764 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/0bdc598fc2de4f51b6e4def2169f8764 2024-12-11T20:36:34,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/0bdc598fc2de4f51b6e4def2169f8764, entries=150, sequenceid=392, filesize=12.0 K 2024-12-11T20:36:34,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for da1a8ec404804dae6a302e6015dbd206 in 461ms, sequenceid=392, compaction requested=true 2024-12-11T20:36:34,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:34,520 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:34,521 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:34,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:34,523 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:34,523 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/A is initiating minor compaction (all files) 2024-12-11T20:36:34,523 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/A in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:34,523 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3d4f86904ca74bf1a90590e5bebd7b36, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b8a68b53f73742c0993b7fd95bfa3478, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f95c2664676f46fd9d78ece21c2265a8] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=39.0 K 2024-12-11T20:36:34,523 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:34,523 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/B is initiating minor compaction (all files) 2024-12-11T20:36:34,523 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/B in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:34,523 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c553bca5aed84fbbabb833a9ab2edd52, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/671226a1458449b4904bdb1802cc8438, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/1a0cbebb3e944221813cc84551b67cb3] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=36.6 K 2024-12-11T20:36:34,523 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c553bca5aed84fbbabb833a9ab2edd52, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733949391393 2024-12-11T20:36:34,523 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d4f86904ca74bf1a90590e5bebd7b36, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733949391393 2024-12-11T20:36:34,524 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 671226a1458449b4904bdb1802cc8438, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733949391467 2024-12-11T20:36:34,524 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8a68b53f73742c0993b7fd95bfa3478, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733949391467 2024-12-11T20:36:34,524 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a0cbebb3e944221813cc84551b67cb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733949393704 2024-12-11T20:36:34,524 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting f95c2664676f46fd9d78ece21c2265a8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733949393704 2024-12-11T20:36:34,531 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#B#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:34,532 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/5c426dcc7a3d41da9170a8644eceb726 is 50, key is test_row_0/B:col10/1733949394058/Put/seqid=0 2024-12-11T20:36:34,532 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#A#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:34,533 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/e3fb4a862e574581a09ff60dcd0ed62e is 50, key is test_row_0/A:col10/1733949394058/Put/seqid=0 2024-12-11T20:36:34,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742338_1514 (size=12983) 2024-12-11T20:36:34,557 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/5c426dcc7a3d41da9170a8644eceb726 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/5c426dcc7a3d41da9170a8644eceb726 2024-12-11T20:36:34,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742339_1515 (size=12983) 2024-12-11T20:36:34,566 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/B of da1a8ec404804dae6a302e6015dbd206 into 5c426dcc7a3d41da9170a8644eceb726(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:34,566 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:34,566 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/B, priority=13, startTime=1733949394521; duration=0sec 2024-12-11T20:36:34,566 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:34,566 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:B 2024-12-11T20:36:34,567 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:34,568 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:34,568 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/C is initiating minor compaction (all files) 2024-12-11T20:36:34,568 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/C in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:34,568 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/65d65a74cdef43029625ce7c8bd42c1c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b73a8b2b09fe4b899104aaa3e4b2103f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/0bdc598fc2de4f51b6e4def2169f8764] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=36.6 K 2024-12-11T20:36:34,569 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d65a74cdef43029625ce7c8bd42c1c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1733949391393 2024-12-11T20:36:34,569 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting b73a8b2b09fe4b899104aaa3e4b2103f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733949391467 2024-12-11T20:36:34,569 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bdc598fc2de4f51b6e4def2169f8764, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733949393704 2024-12-11T20:36:34,570 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/e3fb4a862e574581a09ff60dcd0ed62e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/e3fb4a862e574581a09ff60dcd0ed62e 2024-12-11T20:36:34,574 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/A of da1a8ec404804dae6a302e6015dbd206 into e3fb4a862e574581a09ff60dcd0ed62e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:34,574 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:34,574 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/A, priority=13, startTime=1733949394520; duration=0sec 2024-12-11T20:36:34,574 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:34,574 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:A 2024-12-11T20:36:34,578 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:34,579 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/bdb88ebc7e974274bf4691bf10d68af4 is 50, key is test_row_0/C:col10/1733949394058/Put/seqid=0 2024-12-11T20:36:34,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742340_1516 (size=12983) 2024-12-11T20:36:34,615 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/bdb88ebc7e974274bf4691bf10d68af4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/bdb88ebc7e974274bf4691bf10d68af4 2024-12-11T20:36:34,622 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/C of da1a8ec404804dae6a302e6015dbd206 into bdb88ebc7e974274bf4691bf10d68af4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:34,622 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:34,622 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/C, priority=13, startTime=1733949394521; duration=0sec 2024-12-11T20:36:34,622 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:34,622 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:C 2024-12-11T20:36:34,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-11T20:36:34,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:34,639 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T20:36:34,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:34,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:34,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:34,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:34,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:34,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:34,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/304dfdbacf2a47b29a1f9333543ad084 is 50, key is test_row_0/A:col10/1733949394125/Put/seqid=0 2024-12-11T20:36:34,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742341_1517 (size=12301) 2024-12-11T20:36:34,651 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/304dfdbacf2a47b29a1f9333543ad084 2024-12-11T20:36:34,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/daafdbdf3be045e786fcfc4ccde5afba is 50, key is test_row_0/B:col10/1733949394125/Put/seqid=0 2024-12-11T20:36:34,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742342_1518 (size=12301) 2024-12-11T20:36:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:34,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-11T20:36:34,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 368 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949454843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:34,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:34,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:34,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:34,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 369 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949454957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:35,065 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/daafdbdf3be045e786fcfc4ccde5afba 2024-12-11T20:36:35,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/75393b8fbfbc42068c9a3737eb30a9bc is 50, key is test_row_0/C:col10/1733949394125/Put/seqid=0 2024-12-11T20:36:35,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742343_1519 (size=12301) 2024-12-11T20:36:35,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:35,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 370 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949455166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:35,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-11T20:36:35,475 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/75393b8fbfbc42068c9a3737eb30a9bc 2024-12-11T20:36:35,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/304dfdbacf2a47b29a1f9333543ad084 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/304dfdbacf2a47b29a1f9333543ad084 2024-12-11T20:36:35,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 371 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949455477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:35,481 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/304dfdbacf2a47b29a1f9333543ad084, entries=150, sequenceid=412, filesize=12.0 K 2024-12-11T20:36:35,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/daafdbdf3be045e786fcfc4ccde5afba as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/daafdbdf3be045e786fcfc4ccde5afba 2024-12-11T20:36:35,484 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/daafdbdf3be045e786fcfc4ccde5afba, entries=150, sequenceid=412, filesize=12.0 K 2024-12-11T20:36:35,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/75393b8fbfbc42068c9a3737eb30a9bc as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/75393b8fbfbc42068c9a3737eb30a9bc 2024-12-11T20:36:35,488 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/75393b8fbfbc42068c9a3737eb30a9bc, entries=150, sequenceid=412, filesize=12.0 K 2024-12-11T20:36:35,489 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for da1a8ec404804dae6a302e6015dbd206 in 850ms, sequenceid=412, compaction requested=false 2024-12-11T20:36:35,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:35,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:35,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-11T20:36:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-11T20:36:35,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-11T20:36:35,493 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3110 sec 2024-12-11T20:36:35,495 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3160 sec 2024-12-11T20:36:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:35,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:36:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:35,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/5509c464aaf848b68ecd52efcd245d3b is 50, key is test_row_0/A:col10/1733949394841/Put/seqid=0 2024-12-11T20:36:35,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742344_1520 (size=14741) 2024-12-11T20:36:36,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:36,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 386 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949456060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:36,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:36,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 387 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949456167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:36,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-11T20:36:36,327 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:36,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-11T20:36:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-11T20:36:36,328 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:36,329 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:36,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:36,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 388 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949456377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:36,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/5509c464aaf848b68ecd52efcd245d3b 2024-12-11T20:36:36,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/aea2417b70624945b95688d1b1cb6215 is 50, key is test_row_0/B:col10/1733949394841/Put/seqid=0 2024-12-11T20:36:36,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742345_1521 (size=12301) 2024-12-11T20:36:36,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/aea2417b70624945b95688d1b1cb6215 2024-12-11T20:36:36,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/69ef4db4799b4d1fb880a8f1feb62ca9 is 50, key is test_row_0/C:col10/1733949394841/Put/seqid=0 2024-12-11T20:36:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-11T20:36:36,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742346_1522 (size=12301) 2024-12-11T20:36:36,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-11T20:36:36,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:36,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:36,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:36,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:36,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:36,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-11T20:36:36,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:36,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:36,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:36,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:36,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-11T20:36:36,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 389 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949456686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:36,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41222 deadline: 1733949456696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:36,698 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_1' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:36,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41264 deadline: 1733949456767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:36,771 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18283 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:36,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-11T20:36:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:36,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:36,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41246 deadline: 1733949456796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:36,799 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18317 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:36,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41224 deadline: 1733949456807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:36,810 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in da1a8ec404804dae6a302e6015dbd206 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18328 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:36,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:36,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:36,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/69ef4db4799b4d1fb880a8f1feb62ca9 2024-12-11T20:36:36,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/5509c464aaf848b68ecd52efcd245d3b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/5509c464aaf848b68ecd52efcd245d3b 2024-12-11T20:36:36,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/5509c464aaf848b68ecd52efcd245d3b, entries=200, sequenceid=428, filesize=14.4 K 2024-12-11T20:36:36,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/aea2417b70624945b95688d1b1cb6215 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/aea2417b70624945b95688d1b1cb6215 2024-12-11T20:36:36,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/aea2417b70624945b95688d1b1cb6215, entries=150, sequenceid=428, filesize=12.0 K 2024-12-11T20:36:36,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/69ef4db4799b4d1fb880a8f1feb62ca9 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/69ef4db4799b4d1fb880a8f1feb62ca9 2024-12-11T20:36:36,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/69ef4db4799b4d1fb880a8f1feb62ca9, entries=150, sequenceid=428, filesize=12.0 K 2024-12-11T20:36:36,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for da1a8ec404804dae6a302e6015dbd206 in 875ms, sequenceid=428, compaction requested=true 2024-12-11T20:36:36,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:36,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:36,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:36,866 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:36,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:36,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:36,866 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:36,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:36,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:36,866 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:36,866 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:36,867 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/A is initiating minor compaction (all files) 2024-12-11T20:36:36,867 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/B is initiating minor compaction (all files) 2024-12-11T20:36:36,867 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/A in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:36,867 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/B in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:36,867 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/e3fb4a862e574581a09ff60dcd0ed62e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/304dfdbacf2a47b29a1f9333543ad084, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/5509c464aaf848b68ecd52efcd245d3b] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=39.1 K 2024-12-11T20:36:36,867 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/5c426dcc7a3d41da9170a8644eceb726, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/daafdbdf3be045e786fcfc4ccde5afba, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/aea2417b70624945b95688d1b1cb6215] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=36.7 K 2024-12-11T20:36:36,867 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3fb4a862e574581a09ff60dcd0ed62e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733949393704 2024-12-11T20:36:36,867 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c426dcc7a3d41da9170a8644eceb726, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733949393704 2024-12-11T20:36:36,867 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 304dfdbacf2a47b29a1f9333543ad084, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733949394125 2024-12-11T20:36:36,868 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting daafdbdf3be045e786fcfc4ccde5afba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733949394125 2024-12-11T20:36:36,868 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5509c464aaf848b68ecd52efcd245d3b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733949394829 2024-12-11T20:36:36,868 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting aea2417b70624945b95688d1b1cb6215, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733949394836 2024-12-11T20:36:36,875 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#A#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:36,875 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#B#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:36,876 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/047dca4305fc41d39e5a36f44baabd85 is 50, key is test_row_0/A:col10/1733949394841/Put/seqid=0 2024-12-11T20:36:36,876 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/01a6fea261a34c2dab5f186143ef57da is 50, key is test_row_0/B:col10/1733949394841/Put/seqid=0 2024-12-11T20:36:36,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742347_1523 (size=13085) 2024-12-11T20:36:36,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742348_1524 (size=13085) 2024-12-11T20:36:36,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-11T20:36:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:36,940 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:36:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:36,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:36,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/740de37d77b449d5a255dc960216ef86 is 50, key is test_row_0/A:col10/1733949396058/Put/seqid=0 2024-12-11T20:36:36,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742349_1525 (size=12301) 2024-12-11T20:36:36,948 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/740de37d77b449d5a255dc960216ef86 2024-12-11T20:36:36,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/6d91d68575d841bf9079178e4402ef0e is 50, key is test_row_0/B:col10/1733949396058/Put/seqid=0 2024-12-11T20:36:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-11T20:36:36,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742350_1526 (size=12301) 2024-12-11T20:36:36,960 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/6d91d68575d841bf9079178e4402ef0e 2024-12-11T20:36:36,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/c4a9be18bca847dcb0e3bc00746cface is 50, key is test_row_0/C:col10/1733949396058/Put/seqid=0 2024-12-11T20:36:36,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742351_1527 (size=12301) 2024-12-11T20:36:37,173 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T20:36:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:37,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:37,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:37,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 403 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949457262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:37,287 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/01a6fea261a34c2dab5f186143ef57da as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/01a6fea261a34c2dab5f186143ef57da 2024-12-11T20:36:37,291 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/B of da1a8ec404804dae6a302e6015dbd206 into 01a6fea261a34c2dab5f186143ef57da(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:37,291 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:37,291 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/B, priority=13, startTime=1733949396866; duration=0sec 2024-12-11T20:36:37,291 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:37,291 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:B 2024-12-11T20:36:37,291 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:37,292 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/047dca4305fc41d39e5a36f44baabd85 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/047dca4305fc41d39e5a36f44baabd85 2024-12-11T20:36:37,292 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:37,292 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/C is initiating minor compaction (all files) 2024-12-11T20:36:37,292 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/C in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:37,292 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/bdb88ebc7e974274bf4691bf10d68af4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/75393b8fbfbc42068c9a3737eb30a9bc, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/69ef4db4799b4d1fb880a8f1feb62ca9] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=36.7 K 2024-12-11T20:36:37,292 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting bdb88ebc7e974274bf4691bf10d68af4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1733949393704 2024-12-11T20:36:37,293 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 75393b8fbfbc42068c9a3737eb30a9bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733949394125 2024-12-11T20:36:37,293 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 69ef4db4799b4d1fb880a8f1feb62ca9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733949394836 2024-12-11T20:36:37,296 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/A of da1a8ec404804dae6a302e6015dbd206 into 047dca4305fc41d39e5a36f44baabd85(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:37,296 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:37,296 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/A, priority=13, startTime=1733949396866; duration=0sec 2024-12-11T20:36:37,296 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:37,296 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:A 2024-12-11T20:36:37,299 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:37,299 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/84fe8f9e59e44e63a62eb26c63400848 is 50, key is test_row_0/C:col10/1733949394841/Put/seqid=0 2024-12-11T20:36:37,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742352_1528 (size=13085) 2024-12-11T20:36:37,375 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=444 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/c4a9be18bca847dcb0e3bc00746cface 2024-12-11T20:36:37,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/740de37d77b449d5a255dc960216ef86 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/740de37d77b449d5a255dc960216ef86 2024-12-11T20:36:37,382 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/740de37d77b449d5a255dc960216ef86, entries=150, sequenceid=444, filesize=12.0 K 2024-12-11T20:36:37,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/6d91d68575d841bf9079178e4402ef0e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/6d91d68575d841bf9079178e4402ef0e 2024-12-11T20:36:37,385 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/6d91d68575d841bf9079178e4402ef0e, entries=150, sequenceid=444, filesize=12.0 K 2024-12-11T20:36:37,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 404 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949457377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:37,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:37,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:37,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/c4a9be18bca847dcb0e3bc00746cface as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c4a9be18bca847dcb0e3bc00746cface 2024-12-11T20:36:37,389 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c4a9be18bca847dcb0e3bc00746cface, entries=150, sequenceid=444, filesize=12.0 K 2024-12-11T20:36:37,389 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for da1a8ec404804dae6a302e6015dbd206 in 449ms, sequenceid=444, compaction requested=false 2024-12-11T20:36:37,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:37,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:37,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-11T20:36:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-11T20:36:37,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-11T20:36:37,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0610 sec 2024-12-11T20:36:37,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.0670 sec 2024-12-11T20:36:37,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-11T20:36:37,467 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:37,467 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:37,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-11T20:36:37,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-11T20:36:37,469 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:37,470 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:37,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-11T20:36:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:37,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T20:36:37,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:37,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:37,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:37,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:37,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:37,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:37,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/fd1727c821ce44799322649526b7dc95 is 50, key is test_row_0/A:col10/1733949397600/Put/seqid=0 2024-12-11T20:36:37,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742353_1529 (size=14741) 2024-12-11T20:36:37,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-11T20:36:37,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:37,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:37,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:37,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 418 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949457684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:37,706 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/84fe8f9e59e44e63a62eb26c63400848 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/84fe8f9e59e44e63a62eb26c63400848 2024-12-11T20:36:37,711 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/C of da1a8ec404804dae6a302e6015dbd206 into 84fe8f9e59e44e63a62eb26c63400848(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:37,711 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:37,711 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/C, priority=13, startTime=1733949396866; duration=0sec 2024-12-11T20:36:37,711 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:37,711 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:C 2024-12-11T20:36:37,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-11T20:36:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:37,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-11T20:36:37,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 419 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949457797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:37,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:37,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-11T20:36:37,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:37,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:37,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:37,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:37,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:38,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 420 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949458007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:38,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:38,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:38,007 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:38,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/fd1727c821ce44799322649526b7dc95 2024-12-11T20:36:38,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/348de8e6445e410dab6172103e819d81 is 50, key is test_row_0/B:col10/1733949397600/Put/seqid=0 2024-12-11T20:36:38,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742354_1530 (size=12301) 2024-12-11T20:36:38,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/348de8e6445e410dab6172103e819d81 2024-12-11T20:36:38,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/d9ae84c1054a4bd4b13b30883a4254ec is 50, key is test_row_0/C:col10/1733949397600/Put/seqid=0 2024-12-11T20:36:38,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742355_1531 (size=12301) 2024-12-11T20:36:38,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-11T20:36:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:38,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-11T20:36:38,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-11T20:36:38,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:38,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:38,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:38,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:38,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:38,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:38,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:38,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 421 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41254 deadline: 1733949458317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 , the old value is region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:38,322 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., hostname=5be53b084ac7,44543,1733949248752, seqNum=2 because the exception is null or not the one we care about 2024-12-11T20:36:38,364 INFO [Thread-2054 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2054. 2024-12-11T20:36:38,364 DEBUG [Thread-2054 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:38,364 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,365 DEBUG [Thread-2054 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,365 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:38,366 INFO [Thread-2050 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2050. 2024-12-11T20:36:38,366 DEBUG [Thread-2050 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:38,366 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,366 DEBUG [Thread-2050 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,366 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:38,369 INFO [Thread-2046 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2046. 2024-12-11T20:36:38,369 DEBUG [Thread-2046 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:38,369 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,369 DEBUG [Thread-2046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,369 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:38,371 INFO [Thread-2052 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2052. 2024-12-11T20:36:38,372 DEBUG [Thread-2052 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:38,372 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,372 DEBUG [Thread-2052 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:38,374 INFO [Thread-2048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2048. 2024-12-11T20:36:38,374 DEBUG [Thread-2048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:38,374 DEBUG [Thread-2048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,374 DEBUG [Thread-2048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:38,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-11T20:36:38,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:38,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:38,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:38,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:38,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:38,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/d9ae84c1054a4bd4b13b30883a4254ec 2024-12-11T20:36:38,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/fd1727c821ce44799322649526b7dc95 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/fd1727c821ce44799322649526b7dc95 2024-12-11T20:36:38,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/fd1727c821ce44799322649526b7dc95, entries=200, sequenceid=463, filesize=14.4 K 2024-12-11T20:36:38,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/348de8e6445e410dab6172103e819d81 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/348de8e6445e410dab6172103e819d81 2024-12-11T20:36:38,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/348de8e6445e410dab6172103e819d81, entries=150, sequenceid=463, filesize=12.0 K 2024-12-11T20:36:38,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/d9ae84c1054a4bd4b13b30883a4254ec as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/d9ae84c1054a4bd4b13b30883a4254ec 2024-12-11T20:36:38,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/d9ae84c1054a4bd4b13b30883a4254ec, entries=150, sequenceid=463, filesize=12.0 K 2024-12-11T20:36:38,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for da1a8ec404804dae6a302e6015dbd206 in 840ms, sequenceid=463, compaction requested=true 2024-12-11T20:36:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store da1a8ec404804dae6a302e6015dbd206:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:38,441 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:38,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:38,441 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:38,442 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:38,442 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:38,442 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/A is initiating minor compaction (all files) 2024-12-11T20:36:38,442 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/B is initiating minor compaction (all files) 2024-12-11T20:36:38,442 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/A in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:38,442 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/B in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:38,442 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/047dca4305fc41d39e5a36f44baabd85, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/740de37d77b449d5a255dc960216ef86, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/fd1727c821ce44799322649526b7dc95] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=39.2 K 2024-12-11T20:36:38,442 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/01a6fea261a34c2dab5f186143ef57da, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/6d91d68575d841bf9079178e4402ef0e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/348de8e6445e410dab6172103e819d81] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=36.8 K 2024-12-11T20:36:38,442 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 047dca4305fc41d39e5a36f44baabd85, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733949394836 2024-12-11T20:36:38,442 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 01a6fea261a34c2dab5f186143ef57da, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733949394836 2024-12-11T20:36:38,442 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 740de37d77b449d5a255dc960216ef86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733949395997 2024-12-11T20:36:38,442 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d91d68575d841bf9079178e4402ef0e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733949395997 2024-12-11T20:36:38,442 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd1727c821ce44799322649526b7dc95, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733949397224 2024-12-11T20:36:38,442 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 348de8e6445e410dab6172103e819d81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733949397224 2024-12-11T20:36:38,447 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#B#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:38,447 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/8f9baee7a4ef4ed2b53f7f21736db62d is 50, key is test_row_0/B:col10/1733949397600/Put/seqid=0 2024-12-11T20:36:38,448 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#A#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:38,448 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/3e8370e14d5243178becc0d3d1d6f1c2 is 50, key is test_row_0/A:col10/1733949397600/Put/seqid=0 2024-12-11T20:36:38,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742357_1533 (size=13187) 2024-12-11T20:36:38,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742356_1532 (size=13187) 2024-12-11T20:36:38,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-11T20:36:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:38,540 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T20:36:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:38,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:38,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/3c527794510e4f9e92bcb2f79e2fd8d0 is 50, key is test_row_0/A:col10/1733949397645/Put/seqid=0 2024-12-11T20:36:38,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742358_1534 (size=12301) 2024-12-11T20:36:38,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-11T20:36:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:38,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. as already flushing 2024-12-11T20:36:38,831 INFO [Thread-2035 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2035. 2024-12-11T20:36:38,832 DEBUG [Thread-2035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:38,832 DEBUG [Thread-2035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,832 DEBUG [Thread-2035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:38,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:38,866 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/8f9baee7a4ef4ed2b53f7f21736db62d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/8f9baee7a4ef4ed2b53f7f21736db62d 2024-12-11T20:36:38,866 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/3e8370e14d5243178becc0d3d1d6f1c2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3e8370e14d5243178becc0d3d1d6f1c2 2024-12-11T20:36:38,870 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/B of da1a8ec404804dae6a302e6015dbd206 into 8f9baee7a4ef4ed2b53f7f21736db62d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:38,870 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/A of da1a8ec404804dae6a302e6015dbd206 into 3e8370e14d5243178becc0d3d1d6f1c2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:38,870 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:38,871 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:38,871 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/B, priority=13, startTime=1733949398441; duration=0sec 2024-12-11T20:36:38,871 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/A, priority=13, startTime=1733949398441; duration=0sec 2024-12-11T20:36:38,871 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:38,871 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:B 2024-12-11T20:36:38,871 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:38,871 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:38,871 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:A 2024-12-11T20:36:38,872 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:38,872 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): da1a8ec404804dae6a302e6015dbd206/C is initiating minor compaction (all files) 2024-12-11T20:36:38,872 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of da1a8ec404804dae6a302e6015dbd206/C in TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:38,872 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/84fe8f9e59e44e63a62eb26c63400848, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c4a9be18bca847dcb0e3bc00746cface, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/d9ae84c1054a4bd4b13b30883a4254ec] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp, totalSize=36.8 K 2024-12-11T20:36:38,872 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 84fe8f9e59e44e63a62eb26c63400848, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733949394836 2024-12-11T20:36:38,873 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c4a9be18bca847dcb0e3bc00746cface, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=444, earliestPutTs=1733949395997 2024-12-11T20:36:38,873 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d9ae84c1054a4bd4b13b30883a4254ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1733949397224 2024-12-11T20:36:38,880 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): da1a8ec404804dae6a302e6015dbd206#C#compaction#458 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:38,880 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/e0010cc4db964bd1b75799a5a1575e52 is 50, key is test_row_0/C:col10/1733949397600/Put/seqid=0 2024-12-11T20:36:38,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742359_1535 (size=13187) 2024-12-11T20:36:38,948 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/3c527794510e4f9e92bcb2f79e2fd8d0 2024-12-11T20:36:38,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/92eb8d903634444b954c64a18dfd276d is 50, key is test_row_0/B:col10/1733949397645/Put/seqid=0 2024-12-11T20:36:38,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742360_1536 (size=12301) 2024-12-11T20:36:39,295 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/e0010cc4db964bd1b75799a5a1575e52 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/e0010cc4db964bd1b75799a5a1575e52 2024-12-11T20:36:39,301 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in da1a8ec404804dae6a302e6015dbd206/C of da1a8ec404804dae6a302e6015dbd206 into e0010cc4db964bd1b75799a5a1575e52(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:39,301 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:39,301 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., storeName=da1a8ec404804dae6a302e6015dbd206/C, priority=13, startTime=1733949398441; duration=0sec 2024-12-11T20:36:39,301 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:39,301 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: da1a8ec404804dae6a302e6015dbd206:C 2024-12-11T20:36:39,366 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/92eb8d903634444b954c64a18dfd276d 2024-12-11T20:36:39,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/fd24da72b7884707a594f008915c0d2e is 50, key is test_row_0/C:col10/1733949397645/Put/seqid=0 2024-12-11T20:36:39,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742361_1537 (size=12301) 2024-12-11T20:36:39,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-11T20:36:39,784 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/fd24da72b7884707a594f008915c0d2e 2024-12-11T20:36:39,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/3c527794510e4f9e92bcb2f79e2fd8d0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3c527794510e4f9e92bcb2f79e2fd8d0 2024-12-11T20:36:39,800 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3c527794510e4f9e92bcb2f79e2fd8d0, entries=150, sequenceid=479, filesize=12.0 K 2024-12-11T20:36:39,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/92eb8d903634444b954c64a18dfd276d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/92eb8d903634444b954c64a18dfd276d 2024-12-11T20:36:39,803 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/92eb8d903634444b954c64a18dfd276d, entries=150, sequenceid=479, filesize=12.0 K 2024-12-11T20:36:39,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/fd24da72b7884707a594f008915c0d2e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/fd24da72b7884707a594f008915c0d2e 2024-12-11T20:36:39,807 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/fd24da72b7884707a594f008915c0d2e, entries=150, sequenceid=479, filesize=12.0 K 2024-12-11T20:36:39,807 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=6.71 KB/6870 for da1a8ec404804dae6a302e6015dbd206 in 1267ms, sequenceid=479, compaction requested=false 2024-12-11T20:36:39,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for da1a8ec404804dae6a302e6015dbd206: 2024-12-11T20:36:39,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:39,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-11T20:36:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-11T20:36:39,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-11T20:36:39,811 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3390 sec 2024-12-11T20:36:39,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.3450 sec 2024-12-11T20:36:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-11T20:36:41,637 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:46,772 INFO [Thread-2043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2043. 2024-12-11T20:36:46,772 DEBUG [Thread-2043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:46,772 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:46,773 DEBUG [Thread-2043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:46,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:46,862 INFO [Thread-2037 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2037. 2024-12-11T20:36:46,862 DEBUG [Thread-2037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:46,862 DEBUG [Thread-2037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:46,862 DEBUG [Thread-2037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:46,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:46,869 INFO [Thread-2039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2039. 2024-12-11T20:36:46,869 DEBUG [Thread-2039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:46,869 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:46,869 DEBUG [Thread-2039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:46,869 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:46,900 INFO [Thread-2041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2041. 2024-12-11T20:36:46,900 DEBUG [Thread-2041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:36:46,900 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:46,901 DEBUG [Thread-2041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:46,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 362 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 11 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1666 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4998 rows 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1675 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5021 rows 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1667 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4998 rows 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1665 2024-12-11T20:36:46,901 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4988 rows 2024-12-11T20:36:46,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1672 2024-12-11T20:36:46,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5016 rows 2024-12-11T20:36:46,902 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T20:36:46,902 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:36:46,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:46,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:46,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:36:46,903 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-11T20:36:46,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:46,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-11T20:36:46,908 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949406908"}]},"ts":"1733949406908"} 2024-12-11T20:36:46,910 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T20:36:46,910 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T20:36:46,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T20:36:46,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da1a8ec404804dae6a302e6015dbd206, UNASSIGN}] 2024-12-11T20:36:46,912 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da1a8ec404804dae6a302e6015dbd206, UNASSIGN 2024-12-11T20:36:46,913 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=da1a8ec404804dae6a302e6015dbd206, regionState=CLOSING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:46,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=da1a8ec404804dae6a302e6015dbd206, UNASSIGN because future has completed 2024-12-11T20:36:46,915 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T20:36:46,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:36:47,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-11T20:36:47,067 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:47,067 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T20:36:47,067 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing da1a8ec404804dae6a302e6015dbd206, disabling compactions & flushes 2024-12-11T20:36:47,067 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:47,067 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:47,067 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. after waiting 0 ms 2024-12-11T20:36:47,067 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:47,067 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing da1a8ec404804dae6a302e6015dbd206 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-11T20:36:47,067 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=A 2024-12-11T20:36:47,067 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:47,067 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=B 2024-12-11T20:36:47,067 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:47,067 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK da1a8ec404804dae6a302e6015dbd206, store=C 2024-12-11T20:36:47,067 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:47,070 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/db73515311734aee906d9d8d36e83309 is 50, key is test_row_1/A:col10/1733949406769/Put/seqid=0 2024-12-11T20:36:47,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742362_1538 (size=9857) 2024-12-11T20:36:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-11T20:36:47,475 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/db73515311734aee906d9d8d36e83309 2024-12-11T20:36:47,490 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/37a4ef9ca1424b2d97d87354f6c872c4 is 50, key is test_row_1/B:col10/1733949406769/Put/seqid=0 2024-12-11T20:36:47,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742363_1539 (size=9857) 2024-12-11T20:36:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-11T20:36:47,895 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/37a4ef9ca1424b2d97d87354f6c872c4 2024-12-11T20:36:47,905 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/eab158ea79884d5eb0cc177f2bb9a67f is 50, key is test_row_1/C:col10/1733949406769/Put/seqid=0 2024-12-11T20:36:47,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742364_1540 (size=9857) 2024-12-11T20:36:48,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-11T20:36:48,311 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/eab158ea79884d5eb0cc177f2bb9a67f 2024-12-11T20:36:48,320 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/A/db73515311734aee906d9d8d36e83309 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/db73515311734aee906d9d8d36e83309 2024-12-11T20:36:48,325 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/db73515311734aee906d9d8d36e83309, entries=100, sequenceid=490, filesize=9.6 K 2024-12-11T20:36:48,326 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/B/37a4ef9ca1424b2d97d87354f6c872c4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/37a4ef9ca1424b2d97d87354f6c872c4 2024-12-11T20:36:48,330 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/37a4ef9ca1424b2d97d87354f6c872c4, entries=100, sequenceid=490, filesize=9.6 K 2024-12-11T20:36:48,331 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/.tmp/C/eab158ea79884d5eb0cc177f2bb9a67f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/eab158ea79884d5eb0cc177f2bb9a67f 2024-12-11T20:36:48,336 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/eab158ea79884d5eb0cc177f2bb9a67f, entries=100, sequenceid=490, filesize=9.6 K 2024-12-11T20:36:48,337 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for da1a8ec404804dae6a302e6015dbd206 in 1269ms, sequenceid=490, compaction requested=true 2024-12-11T20:36:48,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/9206b011e68e46569f9d99cd8ac4013d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b16d7c754e4f49668899740ae0fa5723, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/9e04b72d853841cdb514a8adaefeb547, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/046f7b31152c4c9b93925f9d02745e98, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f10c00be8e284989812fa507419597ec, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/18ae0a9391f548c8a63b05289c7dbd5b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b504f279d82647f7bf351263c978108e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/7aeedb6c51de446c97fb993809f40ab8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/bc4b6f62cba043d29aedc227da574212, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/896b12cd0bfc48749224aa0f2d618ce0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/4b207ab6a7d0447fb2d377e6b270efa8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b93e8d386f5e474a91dbfd58930a7577, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/1d42c4baab8b4c1aa5334c6644254668, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/26df2d95aac24b349bb2d62693f7f744, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/ef1d98dbf8ac43efb41a28fb1f6a770c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/85a462f8520d4b6f8d6924e2f188e2ba, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/a0fa7b7a6751478f919a30696e485c13, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3d4f86904ca74bf1a90590e5bebd7b36, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b8a68b53f73742c0993b7fd95bfa3478, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f95c2664676f46fd9d78ece21c2265a8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/e3fb4a862e574581a09ff60dcd0ed62e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/304dfdbacf2a47b29a1f9333543ad084, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/5509c464aaf848b68ecd52efcd245d3b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/047dca4305fc41d39e5a36f44baabd85, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/740de37d77b449d5a255dc960216ef86, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/fd1727c821ce44799322649526b7dc95] to archive 2024-12-11T20:36:48,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:36:48,340 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/9206b011e68e46569f9d99cd8ac4013d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/9206b011e68e46569f9d99cd8ac4013d 2024-12-11T20:36:48,340 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b16d7c754e4f49668899740ae0fa5723 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b16d7c754e4f49668899740ae0fa5723 2024-12-11T20:36:48,341 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/9e04b72d853841cdb514a8adaefeb547 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/9e04b72d853841cdb514a8adaefeb547 2024-12-11T20:36:48,341 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f10c00be8e284989812fa507419597ec to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f10c00be8e284989812fa507419597ec 2024-12-11T20:36:48,341 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/046f7b31152c4c9b93925f9d02745e98 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/046f7b31152c4c9b93925f9d02745e98 2024-12-11T20:36:48,341 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/18ae0a9391f548c8a63b05289c7dbd5b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/18ae0a9391f548c8a63b05289c7dbd5b 2024-12-11T20:36:48,341 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/7aeedb6c51de446c97fb993809f40ab8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/7aeedb6c51de446c97fb993809f40ab8 2024-12-11T20:36:48,341 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b504f279d82647f7bf351263c978108e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b504f279d82647f7bf351263c978108e 2024-12-11T20:36:48,342 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/bc4b6f62cba043d29aedc227da574212 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/bc4b6f62cba043d29aedc227da574212 2024-12-11T20:36:48,343 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/896b12cd0bfc48749224aa0f2d618ce0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/896b12cd0bfc48749224aa0f2d618ce0 2024-12-11T20:36:48,343 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/4b207ab6a7d0447fb2d377e6b270efa8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/4b207ab6a7d0447fb2d377e6b270efa8 2024-12-11T20:36:48,343 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b93e8d386f5e474a91dbfd58930a7577 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b93e8d386f5e474a91dbfd58930a7577 2024-12-11T20:36:48,343 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/1d42c4baab8b4c1aa5334c6644254668 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/1d42c4baab8b4c1aa5334c6644254668 2024-12-11T20:36:48,344 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/26df2d95aac24b349bb2d62693f7f744 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/26df2d95aac24b349bb2d62693f7f744 2024-12-11T20:36:48,344 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/a0fa7b7a6751478f919a30696e485c13 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/a0fa7b7a6751478f919a30696e485c13 2024-12-11T20:36:48,344 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/ef1d98dbf8ac43efb41a28fb1f6a770c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/ef1d98dbf8ac43efb41a28fb1f6a770c 2024-12-11T20:36:48,344 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/85a462f8520d4b6f8d6924e2f188e2ba to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/85a462f8520d4b6f8d6924e2f188e2ba 2024-12-11T20:36:48,344 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b8a68b53f73742c0993b7fd95bfa3478 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/b8a68b53f73742c0993b7fd95bfa3478 2024-12-11T20:36:48,345 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f95c2664676f46fd9d78ece21c2265a8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/f95c2664676f46fd9d78ece21c2265a8 2024-12-11T20:36:48,345 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3d4f86904ca74bf1a90590e5bebd7b36 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3d4f86904ca74bf1a90590e5bebd7b36 2024-12-11T20:36:48,345 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/e3fb4a862e574581a09ff60dcd0ed62e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/e3fb4a862e574581a09ff60dcd0ed62e 2024-12-11T20:36:48,345 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/304dfdbacf2a47b29a1f9333543ad084 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/304dfdbacf2a47b29a1f9333543ad084 2024-12-11T20:36:48,345 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/740de37d77b449d5a255dc960216ef86 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/740de37d77b449d5a255dc960216ef86 2024-12-11T20:36:48,345 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/047dca4305fc41d39e5a36f44baabd85 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/047dca4305fc41d39e5a36f44baabd85 2024-12-11T20:36:48,346 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/fd1727c821ce44799322649526b7dc95 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/fd1727c821ce44799322649526b7dc95 2024-12-11T20:36:48,347 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/5509c464aaf848b68ecd52efcd245d3b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/5509c464aaf848b68ecd52efcd245d3b 2024-12-11T20:36:48,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/78ff95b7401a4d97b4c4cf2039f09114, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/8804f8fb42da40ee86a62cfb59abf18e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/535252bf251f49848d34114da7ccbc76, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/de8c672096e641e2bc79a5e4ec8ee1aa, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c95d84b0c2264a14b08f018386344b93, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/85071e00d3774c6480af66e7aae45441, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/b81cd0c295084e3d82181141ce1fff21, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/d13f68c513a4400b9f0aa1fbf972b2a4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/467f8d4b8066466e9fe3a12b8be2a7a8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ff02feeea9f24d3da40390e82c6e0f0e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/e486fb57822241f5ad76999a06118408, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c6bd142c3066472eb1a7d83ec5380728, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/70d101a4125a442698ac581cdfdd4e52, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ae832cc635064df09db9b381effe3e4d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/3edaf1d547cd49209e698479b417e627, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/4cdfd690188448bd980adc3238531639, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c553bca5aed84fbbabb833a9ab2edd52, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/132cf559798b481eb4dfe6da7bbd152e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/671226a1458449b4904bdb1802cc8438, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/5c426dcc7a3d41da9170a8644eceb726, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/1a0cbebb3e944221813cc84551b67cb3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/daafdbdf3be045e786fcfc4ccde5afba, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/01a6fea261a34c2dab5f186143ef57da, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/aea2417b70624945b95688d1b1cb6215, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/6d91d68575d841bf9079178e4402ef0e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/348de8e6445e410dab6172103e819d81] to archive 2024-12-11T20:36:48,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:36:48,351 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/8804f8fb42da40ee86a62cfb59abf18e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/8804f8fb42da40ee86a62cfb59abf18e 2024-12-11T20:36:48,351 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/535252bf251f49848d34114da7ccbc76 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/535252bf251f49848d34114da7ccbc76 2024-12-11T20:36:48,351 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/de8c672096e641e2bc79a5e4ec8ee1aa to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/de8c672096e641e2bc79a5e4ec8ee1aa 2024-12-11T20:36:48,351 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/78ff95b7401a4d97b4c4cf2039f09114 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/78ff95b7401a4d97b4c4cf2039f09114 2024-12-11T20:36:48,351 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/d13f68c513a4400b9f0aa1fbf972b2a4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/d13f68c513a4400b9f0aa1fbf972b2a4 2024-12-11T20:36:48,351 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c95d84b0c2264a14b08f018386344b93 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c95d84b0c2264a14b08f018386344b93 2024-12-11T20:36:48,351 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/b81cd0c295084e3d82181141ce1fff21 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/b81cd0c295084e3d82181141ce1fff21 2024-12-11T20:36:48,351 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/85071e00d3774c6480af66e7aae45441 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/85071e00d3774c6480af66e7aae45441 2024-12-11T20:36:48,352 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ff02feeea9f24d3da40390e82c6e0f0e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ff02feeea9f24d3da40390e82c6e0f0e 2024-12-11T20:36:48,352 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/467f8d4b8066466e9fe3a12b8be2a7a8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/467f8d4b8066466e9fe3a12b8be2a7a8 2024-12-11T20:36:48,352 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/e486fb57822241f5ad76999a06118408 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/e486fb57822241f5ad76999a06118408 2024-12-11T20:36:48,352 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/70d101a4125a442698ac581cdfdd4e52 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/70d101a4125a442698ac581cdfdd4e52 2024-12-11T20:36:48,352 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/3edaf1d547cd49209e698479b417e627 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/3edaf1d547cd49209e698479b417e627 2024-12-11T20:36:48,352 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ae832cc635064df09db9b381effe3e4d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/ae832cc635064df09db9b381effe3e4d 2024-12-11T20:36:48,353 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/4cdfd690188448bd980adc3238531639 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/4cdfd690188448bd980adc3238531639 2024-12-11T20:36:48,353 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c6bd142c3066472eb1a7d83ec5380728 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c6bd142c3066472eb1a7d83ec5380728 2024-12-11T20:36:48,353 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c553bca5aed84fbbabb833a9ab2edd52 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/c553bca5aed84fbbabb833a9ab2edd52 2024-12-11T20:36:48,353 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/132cf559798b481eb4dfe6da7bbd152e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/132cf559798b481eb4dfe6da7bbd152e 2024-12-11T20:36:48,354 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/671226a1458449b4904bdb1802cc8438 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/671226a1458449b4904bdb1802cc8438 2024-12-11T20:36:48,354 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/5c426dcc7a3d41da9170a8644eceb726 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/5c426dcc7a3d41da9170a8644eceb726 2024-12-11T20:36:48,354 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/daafdbdf3be045e786fcfc4ccde5afba to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/daafdbdf3be045e786fcfc4ccde5afba 2024-12-11T20:36:48,354 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/01a6fea261a34c2dab5f186143ef57da to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/01a6fea261a34c2dab5f186143ef57da 2024-12-11T20:36:48,354 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/1a0cbebb3e944221813cc84551b67cb3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/1a0cbebb3e944221813cc84551b67cb3 2024-12-11T20:36:48,354 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/aea2417b70624945b95688d1b1cb6215 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/aea2417b70624945b95688d1b1cb6215 2024-12-11T20:36:48,354 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/6d91d68575d841bf9079178e4402ef0e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/6d91d68575d841bf9079178e4402ef0e 2024-12-11T20:36:48,354 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/348de8e6445e410dab6172103e819d81 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/348de8e6445e410dab6172103e819d81 2024-12-11T20:36:48,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/45e62147eada41a6b6094d59f8a72bea, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3aa57af9592c466197db8c52e4b71c06, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/9bbac62f7ed04c4fbcb7a9f1ebd0a90d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/fa647d3c351c439ebf14185be248ed70, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/ee0ec6e561d04032b0edf363dbfb94e3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/887f40b7af7e4d27b13ca189aaf44d86, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/e3685f4c906f4dd580737bf455a050c2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3e278444d9974c9fb4db1de20781314c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/062ee011c75b47c8b3a7b90f58bc4f9e, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/5fd85d4c109748e58a2c8f49f401fe1b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/f5eb63d8f03d4911934f68f5d69b39d9, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/059a4784958744b3ac05047385ab94de, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c39d513784074034b9d52638504bce6d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c6b04d443581407f8d9321a208988e92, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/62e272bdbe974c8890ee527640cb96e1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/97caeb35da0b4767beb5a5d3681346c8, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/65d65a74cdef43029625ce7c8bd42c1c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b95e7bff19fb435c99a23e62e0a7687d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b73a8b2b09fe4b899104aaa3e4b2103f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/bdb88ebc7e974274bf4691bf10d68af4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/0bdc598fc2de4f51b6e4def2169f8764, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/75393b8fbfbc42068c9a3737eb30a9bc, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/84fe8f9e59e44e63a62eb26c63400848, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/69ef4db4799b4d1fb880a8f1feb62ca9, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c4a9be18bca847dcb0e3bc00746cface, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/d9ae84c1054a4bd4b13b30883a4254ec] to archive 2024-12-11T20:36:48,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:36:48,357 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/45e62147eada41a6b6094d59f8a72bea to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/45e62147eada41a6b6094d59f8a72bea 2024-12-11T20:36:48,357 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3aa57af9592c466197db8c52e4b71c06 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3aa57af9592c466197db8c52e4b71c06 2024-12-11T20:36:48,357 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/887f40b7af7e4d27b13ca189aaf44d86 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/887f40b7af7e4d27b13ca189aaf44d86 2024-12-11T20:36:48,358 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/9bbac62f7ed04c4fbcb7a9f1ebd0a90d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/9bbac62f7ed04c4fbcb7a9f1ebd0a90d 2024-12-11T20:36:48,358 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/ee0ec6e561d04032b0edf363dbfb94e3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/ee0ec6e561d04032b0edf363dbfb94e3 2024-12-11T20:36:48,358 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/fa647d3c351c439ebf14185be248ed70 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/fa647d3c351c439ebf14185be248ed70 2024-12-11T20:36:48,358 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3e278444d9974c9fb4db1de20781314c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/3e278444d9974c9fb4db1de20781314c 2024-12-11T20:36:48,358 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/e3685f4c906f4dd580737bf455a050c2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/e3685f4c906f4dd580737bf455a050c2 2024-12-11T20:36:48,359 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/062ee011c75b47c8b3a7b90f58bc4f9e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/062ee011c75b47c8b3a7b90f58bc4f9e 2024-12-11T20:36:48,359 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c6b04d443581407f8d9321a208988e92 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c6b04d443581407f8d9321a208988e92 2024-12-11T20:36:48,359 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/5fd85d4c109748e58a2c8f49f401fe1b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/5fd85d4c109748e58a2c8f49f401fe1b 2024-12-11T20:36:48,359 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/f5eb63d8f03d4911934f68f5d69b39d9 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/f5eb63d8f03d4911934f68f5d69b39d9 2024-12-11T20:36:48,359 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/059a4784958744b3ac05047385ab94de to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/059a4784958744b3ac05047385ab94de 2024-12-11T20:36:48,359 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c39d513784074034b9d52638504bce6d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c39d513784074034b9d52638504bce6d 2024-12-11T20:36:48,359 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/97caeb35da0b4767beb5a5d3681346c8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/97caeb35da0b4767beb5a5d3681346c8 2024-12-11T20:36:48,359 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/62e272bdbe974c8890ee527640cb96e1 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/62e272bdbe974c8890ee527640cb96e1 2024-12-11T20:36:48,360 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/65d65a74cdef43029625ce7c8bd42c1c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/65d65a74cdef43029625ce7c8bd42c1c 2024-12-11T20:36:48,360 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b95e7bff19fb435c99a23e62e0a7687d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b95e7bff19fb435c99a23e62e0a7687d 2024-12-11T20:36:48,360 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b73a8b2b09fe4b899104aaa3e4b2103f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/b73a8b2b09fe4b899104aaa3e4b2103f 2024-12-11T20:36:48,360 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/bdb88ebc7e974274bf4691bf10d68af4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/bdb88ebc7e974274bf4691bf10d68af4 2024-12-11T20:36:48,360 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/75393b8fbfbc42068c9a3737eb30a9bc to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/75393b8fbfbc42068c9a3737eb30a9bc 2024-12-11T20:36:48,360 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/0bdc598fc2de4f51b6e4def2169f8764 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/0bdc598fc2de4f51b6e4def2169f8764 2024-12-11T20:36:48,360 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/84fe8f9e59e44e63a62eb26c63400848 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/84fe8f9e59e44e63a62eb26c63400848 2024-12-11T20:36:48,361 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/69ef4db4799b4d1fb880a8f1feb62ca9 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/69ef4db4799b4d1fb880a8f1feb62ca9 2024-12-11T20:36:48,361 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/d9ae84c1054a4bd4b13b30883a4254ec to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/d9ae84c1054a4bd4b13b30883a4254ec 2024-12-11T20:36:48,361 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c4a9be18bca847dcb0e3bc00746cface to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/c4a9be18bca847dcb0e3bc00746cface 2024-12-11T20:36:48,364 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/recovered.edits/493.seqid, newMaxSeqId=493, maxSeqId=1 2024-12-11T20:36:48,365 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. 2024-12-11T20:36:48,365 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for da1a8ec404804dae6a302e6015dbd206: Waiting for close lock at 1733949407067Running coprocessor pre-close hooks at 1733949407067Disabling compacts and flushes for region at 1733949407067Disabling writes for close at 1733949407067Obtaining lock to block concurrent updates at 1733949407067Preparing flush snapshotting stores in da1a8ec404804dae6a302e6015dbd206 at 1733949407067Finished memstore snapshotting TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733949407067Flushing stores of TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206. at 1733949407068 (+1 ms)Flushing da1a8ec404804dae6a302e6015dbd206/A: creating writer at 1733949407068Flushing da1a8ec404804dae6a302e6015dbd206/A: appending metadata at 1733949407070 (+2 ms)Flushing da1a8ec404804dae6a302e6015dbd206/A: closing flushed file at 1733949407070Flushing da1a8ec404804dae6a302e6015dbd206/B: creating writer at 1733949407486 (+416 ms)Flushing da1a8ec404804dae6a302e6015dbd206/B: appending metadata at 1733949407489 (+3 ms)Flushing da1a8ec404804dae6a302e6015dbd206/B: closing flushed file at 1733949407489Flushing da1a8ec404804dae6a302e6015dbd206/C: creating writer at 1733949407901 (+412 ms)Flushing da1a8ec404804dae6a302e6015dbd206/C: appending metadata at 1733949407904 (+3 ms)Flushing da1a8ec404804dae6a302e6015dbd206/C: closing flushed file at 1733949407904Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2598286f: reopening flushed file at 1733949408319 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@508a3eb7: reopening flushed file at 1733949408325 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43ba5d38: reopening flushed file at 1733949408330 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for da1a8ec404804dae6a302e6015dbd206 in 1269ms, sequenceid=490, compaction requested=true at 1733949408337 (+7 ms)Writing region close event to WAL at 1733949408362 (+25 ms)Running coprocessor post-close hooks at 1733949408365 (+3 ms)Closed at 1733949408365 2024-12-11T20:36:48,366 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:48,367 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=da1a8ec404804dae6a302e6015dbd206, regionState=CLOSED 2024-12-11T20:36:48,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:36:48,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-11T20:36:48,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure da1a8ec404804dae6a302e6015dbd206, server=5be53b084ac7,44543,1733949248752 in 1.4540 sec 2024-12-11T20:36:48,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-11T20:36:48,371 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=da1a8ec404804dae6a302e6015dbd206, UNASSIGN in 1.4580 sec 2024-12-11T20:36:48,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-11T20:36:48,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4620 sec 2024-12-11T20:36:48,374 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949408374"}]},"ts":"1733949408374"} 2024-12-11T20:36:48,375 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T20:36:48,375 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T20:36:48,377 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4720 sec 2024-12-11T20:36:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-11T20:36:49,057 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:49,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-11T20:36:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:49,061 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-11T20:36:49,062 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:49,067 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:49,072 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/recovered.edits] 2024-12-11T20:36:49,117 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3c527794510e4f9e92bcb2f79e2fd8d0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3c527794510e4f9e92bcb2f79e2fd8d0 2024-12-11T20:36:49,117 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3e8370e14d5243178becc0d3d1d6f1c2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/3e8370e14d5243178becc0d3d1d6f1c2 2024-12-11T20:36:49,117 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/db73515311734aee906d9d8d36e83309 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/A/db73515311734aee906d9d8d36e83309 2024-12-11T20:36:49,120 DEBUG [HFileArchiver-32 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/92eb8d903634444b954c64a18dfd276d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/92eb8d903634444b954c64a18dfd276d 2024-12-11T20:36:49,120 DEBUG [HFileArchiver-26 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/8f9baee7a4ef4ed2b53f7f21736db62d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/8f9baee7a4ef4ed2b53f7f21736db62d 2024-12-11T20:36:49,120 DEBUG [HFileArchiver-28 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/37a4ef9ca1424b2d97d87354f6c872c4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/B/37a4ef9ca1424b2d97d87354f6c872c4 2024-12-11T20:36:49,122 DEBUG [HFileArchiver-30 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/e0010cc4db964bd1b75799a5a1575e52 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/e0010cc4db964bd1b75799a5a1575e52 2024-12-11T20:36:49,122 DEBUG [HFileArchiver-29 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/eab158ea79884d5eb0cc177f2bb9a67f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/eab158ea79884d5eb0cc177f2bb9a67f 2024-12-11T20:36:49,122 DEBUG [HFileArchiver-27 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/fd24da72b7884707a594f008915c0d2e to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/C/fd24da72b7884707a594f008915c0d2e 2024-12-11T20:36:49,124 DEBUG [HFileArchiver-31 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/recovered.edits/493.seqid to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206/recovered.edits/493.seqid 2024-12-11T20:36:49,125 DEBUG [HFileArchiver-25 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/da1a8ec404804dae6a302e6015dbd206 2024-12-11T20:36:49,125 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T20:36:49,126 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:49,128 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T20:36:49,129 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T20:36:49,130 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:49,130 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T20:36:49,131 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733949409130"}]},"ts":"9223372036854775807"} 2024-12-11T20:36:49,132 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T20:36:49,132 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => da1a8ec404804dae6a302e6015dbd206, NAME => 'TestAcidGuarantees,,1733949376154.da1a8ec404804dae6a302e6015dbd206.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T20:36:49,132 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T20:36:49,132 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733949409132"}]},"ts":"9223372036854775807"} 2024-12-11T20:36:49,133 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T20:36:49,134 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:49,135 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 76 msec 2024-12-11T20:36:49,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-11T20:36:49,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T20:36:49,167 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:49,175 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=256 (was 258), OpenFileDescriptor=443 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=352 (was 391), ProcessCount=11 (was 11), AvailableMemoryMB=1857 (was 1691) - AvailableMemoryMB LEAK? - 2024-12-11T20:36:49,182 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=256, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=352, ProcessCount=11, AvailableMemoryMB=1857 2024-12-11T20:36:49,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T20:36:49,183 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:36:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:49,185 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-11T20:36:49,185 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:49,185 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-12-11T20:36:49,186 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-11T20:36:49,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T20:36:49,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742365_1541 (size=1061) 2024-12-11T20:36:49,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T20:36:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T20:36:49,595 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043 2024-12-11T20:36:49,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742366_1542 (size=53) 2024-12-11T20:36:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T20:36:50,006 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:36:50,007 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c0305045e5dcee8aa8d5a9ea3acd16e6, disabling compactions & flushes 2024-12-11T20:36:50,007 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:50,007 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:50,007 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. after waiting 0 ms 2024-12-11T20:36:50,007 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:50,007 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:50,007 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c0305045e5dcee8aa8d5a9ea3acd16e6: Waiting for close lock at 1733949410006Disabling compacts and flushes for region at 1733949410006Disabling writes for close at 1733949410007 (+1 ms)Writing region close event to WAL at 1733949410007Closed at 1733949410007 2024-12-11T20:36:50,010 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-11T20:36:50,010 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733949410010"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733949410010"}]},"ts":"1733949410010"} 2024-12-11T20:36:50,013 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-11T20:36:50,014 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-11T20:36:50,014 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949410014"}]},"ts":"1733949410014"} 2024-12-11T20:36:50,016 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-11T20:36:50,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, ASSIGN}] 2024-12-11T20:36:50,018 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, ASSIGN 2024-12-11T20:36:50,019 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, ASSIGN; state=OFFLINE, location=5be53b084ac7,44543,1733949248752; forceNewPlan=false, retain=false 2024-12-11T20:36:50,170 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=c0305045e5dcee8aa8d5a9ea3acd16e6, regionState=OPENING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:50,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, ASSIGN because future has completed 2024-12-11T20:36:50,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:36:50,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T20:36:50,333 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:50,333 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:36:50,333 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,333 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:36:50,334 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,334 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,335 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,337 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:36:50,337 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0305045e5dcee8aa8d5a9ea3acd16e6 columnFamilyName A 2024-12-11T20:36:50,338 DEBUG [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:50,338 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(327): Store=c0305045e5dcee8aa8d5a9ea3acd16e6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:36:50,338 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,340 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:36:50,340 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0305045e5dcee8aa8d5a9ea3acd16e6 columnFamilyName B 2024-12-11T20:36:50,340 DEBUG [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:50,341 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(327): Store=c0305045e5dcee8aa8d5a9ea3acd16e6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:36:50,341 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,343 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:36:50,343 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0305045e5dcee8aa8d5a9ea3acd16e6 columnFamilyName C 2024-12-11T20:36:50,343 DEBUG [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:50,344 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(327): Store=c0305045e5dcee8aa8d5a9ea3acd16e6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:36:50,344 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:50,344 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,346 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,346 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,348 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,348 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,349 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T20:36:50,350 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,352 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-11T20:36:50,352 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened c0305045e5dcee8aa8d5a9ea3acd16e6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65173261, jitterRate=-0.028842732310295105}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T20:36:50,352 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:50,353 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for c0305045e5dcee8aa8d5a9ea3acd16e6: Running coprocessor pre-open hook at 1733949410334Writing region info on filesystem at 1733949410334Initializing all the Stores at 1733949410335 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949410335Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949410335Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949410335Cleaning up temporary data from old regions at 1733949410348 (+13 ms)Running coprocessor post-open hooks at 1733949410352 (+4 ms)Region opened successfully at 1733949410353 (+1 ms) 2024-12-11T20:36:50,354 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., pid=144, masterSystemTime=1733949410328 2024-12-11T20:36:50,355 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:50,355 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:50,356 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=c0305045e5dcee8aa8d5a9ea3acd16e6, regionState=OPEN, openSeqNum=2, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:50,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:36:50,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-11T20:36:50,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 in 183 msec 2024-12-11T20:36:50,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-11T20:36:50,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, ASSIGN in 344 msec 2024-12-11T20:36:50,364 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-11T20:36:50,364 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949410364"}]},"ts":"1733949410364"} 2024-12-11T20:36:50,365 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-11T20:36:50,366 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-11T20:36:50,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1830 sec 2024-12-11T20:36:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-12-11T20:36:51,337 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:51,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24493301, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:51,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:51,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:51,339 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:51,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1759344e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:51,341 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:51,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@940dbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:51,342 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:51,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:51,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:51,345 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-11T20:36:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 5be53b084ac7,39923,1733949248041 2024-12-11T20:36:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@694009dd 2024-12-11T20:36:51,346 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-11T20:36:51,347 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33894, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-11T20:36:51,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-11T20:36:51,349 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-11T20:36:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-11T20:36:51,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742367_1543 (size=1097) 2024-12-11T20:36:51,767 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-11T20:36:51,767 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-11T20:36:51,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T20:36:51,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, REOPEN/MOVE}] 2024-12-11T20:36:51,777 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, REOPEN/MOVE 2024-12-11T20:36:51,778 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c0305045e5dcee8aa8d5a9ea3acd16e6, regionState=CLOSING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:51,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, REOPEN/MOVE because future has completed 2024-12-11T20:36:51,780 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T20:36:51,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:36:51,934 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:51,934 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T20:36:51,935 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing c0305045e5dcee8aa8d5a9ea3acd16e6, disabling compactions & flushes 2024-12-11T20:36:51,935 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:51,935 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:51,935 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. after waiting 0 ms 2024-12-11T20:36:51,935 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:51,942 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-11T20:36:51,943 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:51,943 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for c0305045e5dcee8aa8d5a9ea3acd16e6: Waiting for close lock at 1733949411934Running coprocessor pre-close hooks at 1733949411934Disabling compacts and flushes for region at 1733949411934Disabling writes for close at 1733949411935 (+1 ms)Writing region close event to WAL at 1733949411936 (+1 ms)Running coprocessor post-close hooks at 1733949411943 (+7 ms)Closed at 1733949411943 2024-12-11T20:36:51,943 WARN [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: c0305045e5dcee8aa8d5a9ea3acd16e6 to self. 2024-12-11T20:36:51,945 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:51,946 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c0305045e5dcee8aa8d5a9ea3acd16e6, regionState=CLOSED 2024-12-11T20:36:51,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:36:51,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-11T20:36:51,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 in 169 msec 2024-12-11T20:36:51,952 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, REOPEN/MOVE; state=CLOSED, location=5be53b084ac7,44543,1733949248752; forceNewPlan=false, retain=true 2024-12-11T20:36:52,103 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c0305045e5dcee8aa8d5a9ea3acd16e6, regionState=OPENING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, REOPEN/MOVE because future has completed 2024-12-11T20:36:52,108 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:36:52,269 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,269 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} 2024-12-11T20:36:52,269 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,269 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-11T20:36:52,270 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,270 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,272 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,273 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:36:52,273 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0305045e5dcee8aa8d5a9ea3acd16e6 columnFamilyName A 2024-12-11T20:36:52,275 DEBUG [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:52,275 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(327): Store=c0305045e5dcee8aa8d5a9ea3acd16e6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:36:52,275 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,276 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:36:52,276 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0305045e5dcee8aa8d5a9ea3acd16e6 columnFamilyName B 2024-12-11T20:36:52,276 DEBUG [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:52,276 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(327): Store=c0305045e5dcee8aa8d5a9ea3acd16e6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:36:52,277 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,277 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-11T20:36:52,277 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c0305045e5dcee8aa8d5a9ea3acd16e6 columnFamilyName C 2024-12-11T20:36:52,277 DEBUG [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:52,277 INFO [StoreOpener-c0305045e5dcee8aa8d5a9ea3acd16e6-1 {}] regionserver.HStore(327): Store=c0305045e5dcee8aa8d5a9ea3acd16e6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-11T20:36:52,278 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,278 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,278 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,279 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,280 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,280 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,280 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-11T20:36:52,281 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,282 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened c0305045e5dcee8aa8d5a9ea3acd16e6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61720965, jitterRate=-0.0802859514951706}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-11T20:36:52,282 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,282 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for c0305045e5dcee8aa8d5a9ea3acd16e6: Running coprocessor pre-open hook at 1733949412270Writing region info on filesystem at 1733949412270Initializing all the Stores at 1733949412271 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949412271Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949412271Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733949412271Cleaning up temporary data from old regions at 1733949412280 (+9 ms)Running coprocessor post-open hooks at 1733949412282 (+2 ms)Region opened successfully at 1733949412282 2024-12-11T20:36:52,283 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., pid=149, masterSystemTime=1733949412263 2024-12-11T20:36:52,285 DEBUG [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,285 INFO [RS_OPEN_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,285 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c0305045e5dcee8aa8d5a9ea3acd16e6, regionState=OPEN, openSeqNum=5, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:36:52,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-12-11T20:36:52,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 in 179 msec 2024-12-11T20:36:52,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-11T20:36:52,289 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, REOPEN/MOVE in 513 msec 2024-12-11T20:36:52,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-11T20:36:52,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 519 msec 2024-12-11T20:36:52,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 943 msec 2024-12-11T20:36:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-11T20:36:52,294 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:52,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79446f37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:52,294 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:52,295 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3451fd90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:52,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:52,296 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:52,296 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12db8852, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,297 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ce4f739, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,297 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:52,297 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:52,297 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:52,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2673e1c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:52,298 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33922, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:52,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57260ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54aef7aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,299 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:52,299 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:52,300 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab77703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:52,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:52,301 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a99ce1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,302 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50611da3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,302 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:52,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:52,303 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bf496fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:52,304 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd3843b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e064122, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:52,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:52,305 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70bbaf46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:52,306 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:52,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:52,307 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3adebf70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f99473e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:52,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:52,308 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f723f39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:52,309 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33996, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:52,309 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27890ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34542a4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:52,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:52,311 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5822c562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:52,311 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:52,312 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:52,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20eda8d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f68e08c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:52,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:52,313 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:52,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@568ccae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:52,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:52,314 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:52,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658c8778, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2145d008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:52,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:52,317 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212daeb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:52,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:52,318 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:52,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@795fe0a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41481a10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5be53b084ac7,39923,-1 for getting cluster id 2024-12-11T20:36:52,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-11T20:36:52,321 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2e38c395-40d3-42c4-b93e-358d4c1c19bd' 2024-12-11T20:36:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-11T20:36:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2e38c395-40d3-42c4-b93e-358d4c1c19bd" 2024-12-11T20:36:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd6c00a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5be53b084ac7,39923,-1] 2024-12-11T20:36:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-11T20:36:52,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:36:52,322 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-11T20:36:52,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@174c7d24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-11T20:36:52,323 DEBUG [Thread-2442 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:52,323 DEBUG [Thread-2433 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:52,323 DEBUG [Thread-2427 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:52,323 DEBUG [Thread-2429 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:52,324 DEBUG [Thread-2431 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:52,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:52,324 DEBUG [Thread-2444 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:52,324 DEBUG [Thread-2438 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:52,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:52,325 DEBUG [Thread-2446 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-11T20:36:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:52,325 DEBUG [Thread-2440 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:52,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:52,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:52,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:52,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:52,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:52,326 DEBUG [Thread-2435 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-11T20:36:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:52,326 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:52,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:52,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:36:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:36:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:52,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:52,327 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:52,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T20:36:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:36:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5be53b084ac7,44543,1733949248752, seqNum=-1] 2024-12-11T20:36:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:36:52,328 DEBUG [Thread-2440 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-11T20:36:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:36:52,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:52,329 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:36:52,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-11T20:36:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:36:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:36:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:36:52,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5] 2024-12-11T20:36:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-11T20:36:52,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:36:52,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:52,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:36:52,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:52,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:36:52,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:52,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949472351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949472350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949472353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733949412341, max timestamp=1733949412353Num uniques 859; 2024-12-11T20:36:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-11T20:36:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733949412341, max timestamp=1733949412353Num uniques 859; 2024-12-11T20:36:52,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-11T20:36:52,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733949412341, max timestamp=1733949412353Num uniques 859; 2024-12-11T20:36:52,358 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/5be53b084ac7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-11T20:36:52,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211248d4a30b12d493eb68264aae177bbd1_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949412335/Put/seqid=0 2024-12-11T20:36:52,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742368_1544 (size=12154) 2024-12-11T20:36:52,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57624 deadline: 1733949472376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1733949472377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T20:36:52,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949472456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949472457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949472457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-11T20:36:52,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:52,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57624 deadline: 1733949472487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1733949472487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-11T20:36:52,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:52,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T20:36:52,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949472667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949472667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949472667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57624 deadline: 1733949472696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1733949472696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,700 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,769 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:52,771 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211248d4a30b12d493eb68264aae177bbd1_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211248d4a30b12d493eb68264aae177bbd1_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:52,772 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/a58d8621f42c41d9ad5e659e285f3e05, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:52,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/a58d8621f42c41d9ad5e659e285f3e05 is 175, key is test_row_0/A:col10/1733949412335/Put/seqid=0 2024-12-11T20:36:52,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742369_1545 (size=30955) 2024-12-11T20:36:52,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-11T20:36:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:52,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-11T20:36:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:52,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:52,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T20:36:52,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949472976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949472977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:52,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949472977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:52,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:53,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57624 deadline: 1733949473007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:53,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1733949473007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:53,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-11T20:36:53,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:53,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:53,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:53,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,176 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/a58d8621f42c41d9ad5e659e285f3e05 2024-12-11T20:36:53,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/152a07c17b144a2bbab4d058d85ccded is 50, key is test_row_0/B:col10/1733949412335/Put/seqid=0 2024-12-11T20:36:53,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742370_1546 (size=12001) 2024-12-11T20:36:53,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-11T20:36:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:53,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-11T20:36:53,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:53,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:53,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:53,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T20:36:53,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949473487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:53,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,488 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:53,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:53,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949473487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:53,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:53,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:53,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949473487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:53,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57624 deadline: 1733949473516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,518 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:53,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1733949473516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:53,518 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:53,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-11T20:36:53,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:53,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:53,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:53,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:53,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/152a07c17b144a2bbab4d058d85ccded 2024-12-11T20:36:53,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/5defc68ec7c04599894c22b8b6a31d4c is 50, key is test_row_0/C:col10/1733949412335/Put/seqid=0 2024-12-11T20:36:53,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742371_1547 (size=12001) 2024-12-11T20:36:53,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/5defc68ec7c04599894c22b8b6a31d4c 2024-12-11T20:36:53,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/a58d8621f42c41d9ad5e659e285f3e05 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/a58d8621f42c41d9ad5e659e285f3e05 2024-12-11T20:36:53,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/a58d8621f42c41d9ad5e659e285f3e05, entries=150, sequenceid=15, filesize=30.2 K 2024-12-11T20:36:53,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/152a07c17b144a2bbab4d058d85ccded as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/152a07c17b144a2bbab4d058d85ccded 2024-12-11T20:36:53,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/152a07c17b144a2bbab4d058d85ccded, entries=150, sequenceid=15, filesize=11.7 K 2024-12-11T20:36:53,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/5defc68ec7c04599894c22b8b6a31d4c as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/5defc68ec7c04599894c22b8b6a31d4c 2024-12-11T20:36:53,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/5defc68ec7c04599894c22b8b6a31d4c, entries=150, sequenceid=15, filesize=11.7 K 2024-12-11T20:36:53,647 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.68 KB/129720 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1311ms, sequenceid=15, compaction requested=false 2024-12-11T20:36:53,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:36:53,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-11T20:36:53,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:53,703 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=126.68 KB heapSize=363.23 KB 2024-12-11T20:36:53,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:36:53,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:53,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:36:53,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:53,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:36:53,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:36:53,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121187551799cd044b8386a5a40c8c4c7c42_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949412375/Put/seqid=0 2024-12-11T20:36:53,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742372_1548 (size=12154) 2024-12-11T20:36:53,884 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-11T20:36:54,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:54,159 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121187551799cd044b8386a5a40c8c4c7c42_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121187551799cd044b8386a5a40c8c4c7c42_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:54,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/9d395d55578547b7a527e502e7b2223b, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:54,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/9d395d55578547b7a527e502e7b2223b is 175, key is test_row_0/A:col10/1733949412375/Put/seqid=0 2024-12-11T20:36:54,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742373_1549 (size=30955) 2024-12-11T20:36:54,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T20:36:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:54,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:54,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949474505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949474506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:54,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:54,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949474507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:54,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:54,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:54,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1733949474527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:54,529 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0305045e5dcee8aa8d5a9ea3acd16e6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,530 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:54,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:54,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57624 deadline: 1733949474536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:54,539 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0305045e5dcee8aa8d5a9ea3acd16e6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:54,563 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/9d395d55578547b7a527e502e7b2223b 2024-12-11T20:36:54,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/359bfb05480c4dd588ff93b2ada6fd0f is 50, key is test_row_0/B:col10/1733949412375/Put/seqid=0 2024-12-11T20:36:54,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742374_1550 (size=12001) 2024-12-11T20:36:54,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949474617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:54,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949474617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:54,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949474617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:54,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949474827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:54,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949474827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:54,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949474827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,829 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:54,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:54,972 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/359bfb05480c4dd588ff93b2ada6fd0f 2024-12-11T20:36:54,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/38b376a2ddee493bb980156faae3d480 is 50, key is test_row_0/C:col10/1733949412375/Put/seqid=0 2024-12-11T20:36:54,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742375_1551 (size=12001) 2024-12-11T20:36:55,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949475137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949475137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:55,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949475137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:55,381 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/38b376a2ddee493bb980156faae3d480 2024-12-11T20:36:55,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/9d395d55578547b7a527e502e7b2223b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9d395d55578547b7a527e502e7b2223b 2024-12-11T20:36:55,386 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9d395d55578547b7a527e502e7b2223b, entries=150, sequenceid=51, filesize=30.2 K 2024-12-11T20:36:55,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/359bfb05480c4dd588ff93b2ada6fd0f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/359bfb05480c4dd588ff93b2ada6fd0f 2024-12-11T20:36:55,390 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/359bfb05480c4dd588ff93b2ada6fd0f, entries=150, sequenceid=51, filesize=11.7 K 2024-12-11T20:36:55,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/38b376a2ddee493bb980156faae3d480 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/38b376a2ddee493bb980156faae3d480 2024-12-11T20:36:55,393 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/38b376a2ddee493bb980156faae3d480, entries=150, sequenceid=51, filesize=11.7 K 2024-12-11T20:36:55,396 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~126.68 KB/129720, heapSize ~363.19 KB/371904, currentSize=53.67 KB/54960 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1693ms, sequenceid=51, compaction requested=false 2024-12-11T20:36:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:36:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:55,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-11T20:36:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-11T20:36:55,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-11T20:36:55,399 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0710 sec 2024-12-11T20:36:55,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.0760 sec 2024-12-11T20:36:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:55,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T20:36:55,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:36:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:36:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:36:55,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:55,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412117e09c5c3fefa4d1e8d1e1df51fcd2878_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949414504/Put/seqid=0 2024-12-11T20:36:55,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742376_1552 (size=17034) 2024-12-11T20:36:55,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949475674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949475675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949475675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949475789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949475789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:55,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949475789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949475997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949475997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949475997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:55,999 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:56,060 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:56,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412117e09c5c3fefa4d1e8d1e1df51fcd2878_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117e09c5c3fefa4d1e8d1e1df51fcd2878_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:56,063 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/b2f9f4ab8ad54084a053d094f8c2cc47, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:56,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/b2f9f4ab8ad54084a053d094f8c2cc47 is 175, key is test_row_0/A:col10/1733949414504/Put/seqid=0 2024-12-11T20:36:56,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742377_1553 (size=48139) 2024-12-11T20:36:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949476306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:56,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:56,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949476306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,307 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:56,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949476307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:56,467 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/b2f9f4ab8ad54084a053d094f8c2cc47 2024-12-11T20:36:56,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/ba991c077a544f0b954d18439459cf71 is 50, key is test_row_0/B:col10/1733949414504/Put/seqid=0 2024-12-11T20:36:56,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742378_1554 (size=12001) 2024-12-11T20:36:56,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-11T20:36:56,487 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:56,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-11T20:36:56,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T20:36:56,490 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:56,490 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:56,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:56,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1733949476536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:56,539 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0305045e5dcee8aa8d5a9ea3acd16e6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:56,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,539 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:56,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57624 deadline: 1733949476557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:56,559 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0305045e5dcee8aa8d5a9ea3acd16e6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:36:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T20:36:56,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-11T20:36:56,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:56,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:56,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:56,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:56,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:56,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-11T20:36:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:56,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T20:36:56,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949476817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949476817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:56,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949476817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:56,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/ba991c077a544f0b954d18439459cf71 2024-12-11T20:36:56,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/c690f38492274381a1bc496d43f04ca9 is 50, key is test_row_0/C:col10/1733949414504/Put/seqid=0 2024-12-11T20:36:56,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742379_1555 (size=12001) 2024-12-11T20:36:56,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/c690f38492274381a1bc496d43f04ca9 2024-12-11T20:36:56,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/b2f9f4ab8ad54084a053d094f8c2cc47 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b2f9f4ab8ad54084a053d094f8c2cc47 2024-12-11T20:36:56,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b2f9f4ab8ad54084a053d094f8c2cc47, entries=250, sequenceid=64, filesize=47.0 K 2024-12-11T20:36:56,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/ba991c077a544f0b954d18439459cf71 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/ba991c077a544f0b954d18439459cf71 2024-12-11T20:36:56,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/ba991c077a544f0b954d18439459cf71, entries=150, sequenceid=64, filesize=11.7 K 2024-12-11T20:36:56,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/c690f38492274381a1bc496d43f04ca9 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/c690f38492274381a1bc496d43f04ca9 2024-12-11T20:36:56,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/c690f38492274381a1bc496d43f04ca9, entries=150, sequenceid=64, filesize=11.7 K 2024-12-11T20:36:56,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1249ms, sequenceid=64, compaction requested=true 2024-12-11T20:36:56,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:36:56,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:56,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:56,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:56,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:56,898 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:56,898 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:56,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:56,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:56,899 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:56,899 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:56,899 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/B is initiating minor compaction (all files) 2024-12-11T20:36:56,899 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/A is initiating minor compaction (all files) 2024-12-11T20:36:56,899 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/B in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:56,899 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/A in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:56,899 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/152a07c17b144a2bbab4d058d85ccded, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/359bfb05480c4dd588ff93b2ada6fd0f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/ba991c077a544f0b954d18439459cf71] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=35.2 K 2024-12-11T20:36:56,899 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/a58d8621f42c41d9ad5e659e285f3e05, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9d395d55578547b7a527e502e7b2223b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b2f9f4ab8ad54084a053d094f8c2cc47] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=107.5 K 2024-12-11T20:36:56,899 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:56,899 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/a58d8621f42c41d9ad5e659e285f3e05, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9d395d55578547b7a527e502e7b2223b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b2f9f4ab8ad54084a053d094f8c2cc47] 2024-12-11T20:36:56,899 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 152a07c17b144a2bbab4d058d85ccded, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733949412331 2024-12-11T20:36:56,899 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting a58d8621f42c41d9ad5e659e285f3e05, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733949412331 2024-12-11T20:36:56,900 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d395d55578547b7a527e502e7b2223b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733949412372 2024-12-11T20:36:56,900 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 359bfb05480c4dd588ff93b2ada6fd0f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733949412372 2024-12-11T20:36:56,900 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2f9f4ab8ad54084a053d094f8c2cc47, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733949414504 2024-12-11T20:36:56,900 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting ba991c077a544f0b954d18439459cf71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733949414504 2024-12-11T20:36:56,905 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#B#compaction#473 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:56,906 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/2cbb8b36d0e247b0affc54134b79276b is 50, key is test_row_0/B:col10/1733949414504/Put/seqid=0 2024-12-11T20:36:56,907 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:56,908 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211aee4a615254e44378dbe6b2d2dcbab1a_c0305045e5dcee8aa8d5a9ea3acd16e6 store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:56,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742380_1556 (size=12104) 2024-12-11T20:36:56,910 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211aee4a615254e44378dbe6b2d2dcbab1a_c0305045e5dcee8aa8d5a9ea3acd16e6, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:56,910 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211aee4a615254e44378dbe6b2d2dcbab1a_c0305045e5dcee8aa8d5a9ea3acd16e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:56,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742381_1557 (size=4469) 2024-12-11T20:36:56,915 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/2cbb8b36d0e247b0affc54134b79276b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/2cbb8b36d0e247b0affc54134b79276b 2024-12-11T20:36:56,915 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#A#compaction#474 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:56,916 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/b5498fafe5c442409881b17bc26996f0 is 175, key is test_row_0/A:col10/1733949414504/Put/seqid=0 2024-12-11T20:36:56,920 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/B of c0305045e5dcee8aa8d5a9ea3acd16e6 into 2cbb8b36d0e247b0affc54134b79276b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:56,920 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:36:56,920 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/B, priority=13, startTime=1733949416898; duration=0sec 2024-12-11T20:36:56,920 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:56,920 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:B 2024-12-11T20:36:56,920 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:56,922 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:56,922 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/C is initiating minor compaction (all files) 2024-12-11T20:36:56,922 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/C in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:56,922 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/5defc68ec7c04599894c22b8b6a31d4c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/38b376a2ddee493bb980156faae3d480, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/c690f38492274381a1bc496d43f04ca9] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=35.2 K 2024-12-11T20:36:56,922 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 5defc68ec7c04599894c22b8b6a31d4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733949412331 2024-12-11T20:36:56,923 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b376a2ddee493bb980156faae3d480, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733949412372 2024-12-11T20:36:56,923 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting c690f38492274381a1bc496d43f04ca9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733949414504 2024-12-11T20:36:56,929 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#C#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:56,929 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/8c5b155b8ef64acda7d7453a1cb55da9 is 50, key is test_row_0/C:col10/1733949414504/Put/seqid=0 2024-12-11T20:36:56,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742382_1558 (size=31058) 2024-12-11T20:36:56,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742383_1559 (size=12104) 2024-12-11T20:36:56,941 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/8c5b155b8ef64acda7d7453a1cb55da9 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/8c5b155b8ef64acda7d7453a1cb55da9 2024-12-11T20:36:56,945 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/C of c0305045e5dcee8aa8d5a9ea3acd16e6 into 8c5b155b8ef64acda7d7453a1cb55da9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:56,945 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:36:56,945 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/C, priority=13, startTime=1733949416898; duration=0sec 2024-12-11T20:36:56,945 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:56,945 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:C 2024-12-11T20:36:56,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-11T20:36:56,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:56,947 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T20:36:56,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:36:56,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:56,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:36:56,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:56,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:36:56,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:56,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115e92ae7eba2d4a3db34673454d1e5713_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949415674/Put/seqid=0 2024-12-11T20:36:56,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742384_1560 (size=12154) 2024-12-11T20:36:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T20:36:57,338 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/b5498fafe5c442409881b17bc26996f0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b5498fafe5c442409881b17bc26996f0 2024-12-11T20:36:57,341 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/A of c0305045e5dcee8aa8d5a9ea3acd16e6 into b5498fafe5c442409881b17bc26996f0(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:57,341 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:36:57,341 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/A, priority=13, startTime=1733949416898; duration=0sec 2024-12-11T20:36:57,341 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:57,341 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:A 2024-12-11T20:36:57,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:57,359 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115e92ae7eba2d4a3db34673454d1e5713_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115e92ae7eba2d4a3db34673454d1e5713_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:57,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/95fccb9c293c4e03936191359c622e6a, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:57,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/95fccb9c293c4e03936191359c622e6a is 175, key is test_row_0/A:col10/1733949415674/Put/seqid=0 2024-12-11T20:36:57,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742385_1561 (size=30955) 2024-12-11T20:36:57,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T20:36:57,764 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/95fccb9c293c4e03936191359c622e6a 2024-12-11T20:36:57,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/590d06ed92e94df388491ac6121272cf is 50, key is test_row_0/B:col10/1733949415674/Put/seqid=0 2024-12-11T20:36:57,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742386_1562 (size=12001) 2024-12-11T20:36:57,773 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/590d06ed92e94df388491ac6121272cf 2024-12-11T20:36:57,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/e22ab6a407e94903acbe58a3adfb3910 is 50, key is test_row_0/C:col10/1733949415674/Put/seqid=0 2024-12-11T20:36:57,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742387_1563 (size=12001) 2024-12-11T20:36:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:57,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:57,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949477846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:57,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949477847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:57,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949477848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:57,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:57,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949477957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949477957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:57,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949477957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:57,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949478166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949478167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949478167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,185 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/e22ab6a407e94903acbe58a3adfb3910 2024-12-11T20:36:58,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/95fccb9c293c4e03936191359c622e6a as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/95fccb9c293c4e03936191359c622e6a 2024-12-11T20:36:58,191 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/95fccb9c293c4e03936191359c622e6a, entries=150, sequenceid=86, filesize=30.2 K 2024-12-11T20:36:58,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/590d06ed92e94df388491ac6121272cf as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/590d06ed92e94df388491ac6121272cf 2024-12-11T20:36:58,194 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/590d06ed92e94df388491ac6121272cf, entries=150, sequenceid=86, filesize=11.7 K 2024-12-11T20:36:58,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/e22ab6a407e94903acbe58a3adfb3910 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/e22ab6a407e94903acbe58a3adfb3910 2024-12-11T20:36:58,197 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/e22ab6a407e94903acbe58a3adfb3910, entries=150, sequenceid=86, filesize=11.7 K 2024-12-11T20:36:58,198 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1251ms, sequenceid=86, compaction requested=false 2024-12-11T20:36:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:36:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:58,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-11T20:36:58,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-11T20:36:58,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-11T20:36:58,200 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7090 sec 2024-12-11T20:36:58,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7130 sec 2024-12-11T20:36:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:58,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T20:36:58,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:36:58,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:58,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:36:58,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:58,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:36:58,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:58,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121105618ff3e8d444088918fde761e5a901_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949418479/Put/seqid=0 2024-12-11T20:36:58,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742388_1564 (size=17034) 2024-12-11T20:36:58,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949478496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949478497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949478498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949478606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949478606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949478606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-11T20:36:58,637 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:36:58,637 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:36:58,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-11T20:36:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-11T20:36:58,639 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:36:58,639 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:36:58,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:36:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-11T20:36:58,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-11T20:36:58,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:58,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:58,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:58,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:58,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:58,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949478816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949478816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949478817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:58,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:58,890 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:58,892 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121105618ff3e8d444088918fde761e5a901_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121105618ff3e8d444088918fde761e5a901_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:58,893 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/cb97c8d4f9614957b0e7d23ec16cc016, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:58,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/cb97c8d4f9614957b0e7d23ec16cc016 is 175, key is test_row_0/A:col10/1733949418479/Put/seqid=0 2024-12-11T20:36:58,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742389_1565 (size=48139) 2024-12-11T20:36:58,896 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/cb97c8d4f9614957b0e7d23ec16cc016 2024-12-11T20:36:58,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/64d9fd8c738d41678a1526ca92a3e2ce is 50, key is test_row_0/B:col10/1733949418479/Put/seqid=0 2024-12-11T20:36:58,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742390_1566 (size=12001) 2024-12-11T20:36:58,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/64d9fd8c738d41678a1526ca92a3e2ce 2024-12-11T20:36:58,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/4957cb75921742d3959398c8426fc63d is 50, key is test_row_0/C:col10/1733949418479/Put/seqid=0 2024-12-11T20:36:58,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742391_1567 (size=12001) 2024-12-11T20:36:58,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-11T20:36:58,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:58,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:58,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:58,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:58,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:58,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:58,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-11T20:36:59,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-11T20:36:59,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:59,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:59,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:59,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:59,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:59,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949479127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949479127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949479127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:59,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-11T20:36:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:59,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:59,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:59,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:36:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-11T20:36:59,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/4957cb75921742d3959398c8426fc63d 2024-12-11T20:36:59,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/cb97c8d4f9614957b0e7d23ec16cc016 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/cb97c8d4f9614957b0e7d23ec16cc016 2024-12-11T20:36:59,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/cb97c8d4f9614957b0e7d23ec16cc016, entries=250, sequenceid=102, filesize=47.0 K 2024-12-11T20:36:59,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/64d9fd8c738d41678a1526ca92a3e2ce as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/64d9fd8c738d41678a1526ca92a3e2ce 2024-12-11T20:36:59,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/64d9fd8c738d41678a1526ca92a3e2ce, entries=150, sequenceid=102, filesize=11.7 K 2024-12-11T20:36:59,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/4957cb75921742d3959398c8426fc63d as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/4957cb75921742d3959398c8426fc63d 2024-12-11T20:36:59,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/4957cb75921742d3959398c8426fc63d, entries=150, sequenceid=102, filesize=11.7 K 2024-12-11T20:36:59,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 844ms, sequenceid=102, compaction requested=true 2024-12-11T20:36:59,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:36:59,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:36:59,324 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:59,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:59,324 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:59,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:36:59,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:59,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:36:59,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:59,327 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:59,327 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:59,327 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/B is initiating minor compaction (all files) 2024-12-11T20:36:59,327 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/A is initiating minor compaction (all files) 2024-12-11T20:36:59,327 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/A in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:59,327 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/B in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:59,327 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/2cbb8b36d0e247b0affc54134b79276b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/590d06ed92e94df388491ac6121272cf, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/64d9fd8c738d41678a1526ca92a3e2ce] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=35.3 K 2024-12-11T20:36:59,327 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b5498fafe5c442409881b17bc26996f0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/95fccb9c293c4e03936191359c622e6a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/cb97c8d4f9614957b0e7d23ec16cc016] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=107.6 K 2024-12-11T20:36:59,327 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:59,327 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b5498fafe5c442409881b17bc26996f0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/95fccb9c293c4e03936191359c622e6a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/cb97c8d4f9614957b0e7d23ec16cc016] 2024-12-11T20:36:59,328 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cbb8b36d0e247b0affc54134b79276b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733949414504 2024-12-11T20:36:59,328 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5498fafe5c442409881b17bc26996f0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733949414504 2024-12-11T20:36:59,328 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 590d06ed92e94df388491ac6121272cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733949415664 2024-12-11T20:36:59,328 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95fccb9c293c4e03936191359c622e6a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733949415664 2024-12-11T20:36:59,328 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 64d9fd8c738d41678a1526ca92a3e2ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733949417847 2024-12-11T20:36:59,328 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb97c8d4f9614957b0e7d23ec16cc016, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733949417844 2024-12-11T20:36:59,332 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:59,333 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#B#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:59,334 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/9fb344c43801487d8d3f1b8d9cd9ba17 is 50, key is test_row_0/B:col10/1733949418479/Put/seqid=0 2024-12-11T20:36:59,334 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412115935ca64cdb34ff3a7af444e04d6eb51_c0305045e5dcee8aa8d5a9ea3acd16e6 store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:59,336 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412115935ca64cdb34ff3a7af444e04d6eb51_c0305045e5dcee8aa8d5a9ea3acd16e6, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:59,336 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412115935ca64cdb34ff3a7af444e04d6eb51_c0305045e5dcee8aa8d5a9ea3acd16e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:59,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742392_1568 (size=12207) 2024-12-11T20:36:59,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742393_1569 (size=4469) 2024-12-11T20:36:59,339 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#A#compaction#483 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:59,340 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/03d548a357b3449ba11d041e680e7e5f is 175, key is test_row_0/A:col10/1733949418479/Put/seqid=0 2024-12-11T20:36:59,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742394_1570 (size=31161) 2024-12-11T20:36:59,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-11T20:36:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:59,402 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T20:36:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:36:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:36:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:36:59,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:36:59,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412117afb9eb7fb02459e93c06a0922105f44_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949418497/Put/seqid=0 2024-12-11T20:36:59,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742395_1571 (size=12154) 2024-12-11T20:36:59,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:59,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:36:59,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949479673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:59,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949479674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949479675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:59,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:59,741 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/9fb344c43801487d8d3f1b8d9cd9ba17 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/9fb344c43801487d8d3f1b8d9cd9ba17 2024-12-11T20:36:59,745 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/B of c0305045e5dcee8aa8d5a9ea3acd16e6 into 9fb344c43801487d8d3f1b8d9cd9ba17(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:59,745 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:36:59,745 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/B, priority=13, startTime=1733949419324; duration=0sec 2024-12-11T20:36:59,745 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:36:59,745 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:B 2024-12-11T20:36:59,746 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:36:59,747 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/03d548a357b3449ba11d041e680e7e5f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/03d548a357b3449ba11d041e680e7e5f 2024-12-11T20:36:59,747 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:36:59,747 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/C is initiating minor compaction (all files) 2024-12-11T20:36:59,747 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/C in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:36:59,747 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/8c5b155b8ef64acda7d7453a1cb55da9, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/e22ab6a407e94903acbe58a3adfb3910, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/4957cb75921742d3959398c8426fc63d] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=35.3 K 2024-12-11T20:36:59,748 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c5b155b8ef64acda7d7453a1cb55da9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1733949414504 2024-12-11T20:36:59,748 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting e22ab6a407e94903acbe58a3adfb3910, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733949415664 2024-12-11T20:36:59,748 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 4957cb75921742d3959398c8426fc63d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733949417847 2024-12-11T20:36:59,751 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/A of c0305045e5dcee8aa8d5a9ea3acd16e6 into 03d548a357b3449ba11d041e680e7e5f(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:59,751 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:36:59,751 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/A, priority=13, startTime=1733949419324; duration=0sec 2024-12-11T20:36:59,751 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:59,751 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:A 2024-12-11T20:36:59,754 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#C#compaction#485 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:36:59,754 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/be0fa8b9c90841968f3631e25bcade8b is 50, key is test_row_0/C:col10/1733949418479/Put/seqid=0 2024-12-11T20:36:59,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742396_1572 (size=12207) 2024-12-11T20:36:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-11T20:36:59,781 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/be0fa8b9c90841968f3631e25bcade8b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/be0fa8b9c90841968f3631e25bcade8b 2024-12-11T20:36:59,785 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/C of c0305045e5dcee8aa8d5a9ea3acd16e6 into be0fa8b9c90841968f3631e25bcade8b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:36:59,785 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:36:59,785 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/C, priority=13, startTime=1733949419324; duration=0sec 2024-12-11T20:36:59,785 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:36:59,785 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:C 2024-12-11T20:36:59,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949479787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949479787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949479787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:59,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:59,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:36:59,813 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412117afb9eb7fb02459e93c06a0922105f44_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117afb9eb7fb02459e93c06a0922105f44_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:36:59,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/e7d68b5ea7584fbd85fc155fd3fe19f1, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:36:59,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/e7d68b5ea7584fbd85fc155fd3fe19f1 is 175, key is test_row_0/A:col10/1733949418497/Put/seqid=0 2024-12-11T20:36:59,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742397_1573 (size=30955) 2024-12-11T20:36:59,825 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/e7d68b5ea7584fbd85fc155fd3fe19f1 2024-12-11T20:36:59,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/629de55302334dc69bda53bf900ad1d0 is 50, key is test_row_0/B:col10/1733949418497/Put/seqid=0 2024-12-11T20:36:59,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742398_1574 (size=12001) 2024-12-11T20:36:59,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949479996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949479996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:36:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949479996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:36:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:59,998 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:36:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:36:59,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:00,234 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/629de55302334dc69bda53bf900ad1d0 2024-12-11T20:37:00,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/f03ce7e164ea42ef8aa4d1b3c3ba04b1 is 50, key is test_row_0/C:col10/1733949418497/Put/seqid=0 2024-12-11T20:37:00,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742399_1575 (size=12001) 2024-12-11T20:37:00,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949480307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949480307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,310 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:00,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949480307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:00,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1733949480566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:00,568 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0305045e5dcee8aa8d5a9ea3acd16e6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:37:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,569 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:00,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57624 deadline: 1733949480576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:00,578 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0305045e5dcee8aa8d5a9ea3acd16e6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:37:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:00,642 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/f03ce7e164ea42ef8aa4d1b3c3ba04b1 2024-12-11T20:37:00,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/e7d68b5ea7584fbd85fc155fd3fe19f1 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e7d68b5ea7584fbd85fc155fd3fe19f1 2024-12-11T20:37:00,647 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e7d68b5ea7584fbd85fc155fd3fe19f1, entries=150, sequenceid=119, filesize=30.2 K 2024-12-11T20:37:00,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/629de55302334dc69bda53bf900ad1d0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/629de55302334dc69bda53bf900ad1d0 2024-12-11T20:37:00,650 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/629de55302334dc69bda53bf900ad1d0, entries=150, sequenceid=119, filesize=11.7 K 2024-12-11T20:37:00,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/f03ce7e164ea42ef8aa4d1b3c3ba04b1 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f03ce7e164ea42ef8aa4d1b3c3ba04b1 2024-12-11T20:37:00,654 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f03ce7e164ea42ef8aa4d1b3c3ba04b1, entries=150, sequenceid=119, filesize=11.7 K 2024-12-11T20:37:00,654 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1252ms, sequenceid=119, compaction requested=false 2024-12-11T20:37:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:00,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-11T20:37:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-11T20:37:00,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-11T20:37:00,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0170 sec 2024-12-11T20:37:00,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.0190 sec 2024-12-11T20:37:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-11T20:37:00,787 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:37:00,788 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:37:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-11T20:37:00,789 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:37:00,790 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:37:00,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:37:00,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T20:37:00,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:00,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:37:00,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:37:00,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:00,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:37:00,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:00,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:37:00,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:00,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e2802a1aee4f42a797c044375fe5aef0_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949419672/Put/seqid=0 2024-12-11T20:37:00,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742400_1576 (size=12304) 2024-12-11T20:37:00,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:00,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949480863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:00,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949480864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949480864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T20:37:00,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:00,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:00,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:00,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:00,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949480977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:00,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949480977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949480977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:00,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:01,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T20:37:01,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949481187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:01,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949481187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:01,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949481187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:01,234 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:01,237 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211e2802a1aee4f42a797c044375fe5aef0_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e2802a1aee4f42a797c044375fe5aef0_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:01,238 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5098f7ed372a4f02a51fe15d59244f90, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:01,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5098f7ed372a4f02a51fe15d59244f90 is 175, key is test_row_0/A:col10/1733949419672/Put/seqid=0 2024-12-11T20:37:01,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742401_1577 (size=31105) 2024-12-11T20:37:01,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:01,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:01,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T20:37:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949481496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949481497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:01,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949481497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:01,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:01,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,642 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5098f7ed372a4f02a51fe15d59244f90 2024-12-11T20:37:01,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/640e85a0a4294cc0b30b6fbb29cf84d6 is 50, key is test_row_0/B:col10/1733949419672/Put/seqid=0 2024-12-11T20:37:01,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742402_1578 (size=12151) 2024-12-11T20:37:01,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:01,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T20:37:02,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949482006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:02,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949482007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:02,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949482007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:02,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:02,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:02,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/640e85a0a4294cc0b30b6fbb29cf84d6 2024-12-11T20:37:02,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/75d1206cc9ab417e9361242fcaa1ff9b is 50, key is test_row_0/C:col10/1733949419672/Put/seqid=0 2024-12-11T20:37:02,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742403_1579 (size=12151) 2024-12-11T20:37:02,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:02,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:02,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:02,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:02,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/75d1206cc9ab417e9361242fcaa1ff9b 2024-12-11T20:37:02,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5098f7ed372a4f02a51fe15d59244f90 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5098f7ed372a4f02a51fe15d59244f90 2024-12-11T20:37:02,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:02,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:02,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5098f7ed372a4f02a51fe15d59244f90, entries=150, sequenceid=138, filesize=30.4 K 2024-12-11T20:37:02,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/640e85a0a4294cc0b30b6fbb29cf84d6 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/640e85a0a4294cc0b30b6fbb29cf84d6 2024-12-11T20:37:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:02,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/640e85a0a4294cc0b30b6fbb29cf84d6, entries=150, sequenceid=138, filesize=11.9 K 2024-12-11T20:37:02,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/75d1206cc9ab417e9361242fcaa1ff9b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/75d1206cc9ab417e9361242fcaa1ff9b 2024-12-11T20:37:02,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/75d1206cc9ab417e9361242fcaa1ff9b, entries=150, sequenceid=138, filesize=11.9 K 2024-12-11T20:37:02,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1662ms, sequenceid=138, compaction requested=true 2024-12-11T20:37:02,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:02,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:37:02,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:02,480 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:37:02,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:37:02,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:02,480 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:37:02,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:37:02,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:37:02,481 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:37:02,481 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:37:02,481 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/B is initiating minor compaction (all files) 2024-12-11T20:37:02,481 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/A is initiating minor compaction (all files) 2024-12-11T20:37:02,481 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/B in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,481 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/A in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,481 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/9fb344c43801487d8d3f1b8d9cd9ba17, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/629de55302334dc69bda53bf900ad1d0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/640e85a0a4294cc0b30b6fbb29cf84d6] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=35.5 K 2024-12-11T20:37:02,481 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/03d548a357b3449ba11d041e680e7e5f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e7d68b5ea7584fbd85fc155fd3fe19f1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5098f7ed372a4f02a51fe15d59244f90] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=91.0 K 2024-12-11T20:37:02,481 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,481 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/03d548a357b3449ba11d041e680e7e5f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e7d68b5ea7584fbd85fc155fd3fe19f1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5098f7ed372a4f02a51fe15d59244f90] 2024-12-11T20:37:02,482 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03d548a357b3449ba11d041e680e7e5f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733949417847 2024-12-11T20:37:02,482 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb344c43801487d8d3f1b8d9cd9ba17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733949417847 2024-12-11T20:37:02,482 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 629de55302334dc69bda53bf900ad1d0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733949418490 2024-12-11T20:37:02,482 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7d68b5ea7584fbd85fc155fd3fe19f1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733949418490 2024-12-11T20:37:02,482 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 640e85a0a4294cc0b30b6fbb29cf84d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733949419672 2024-12-11T20:37:02,482 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5098f7ed372a4f02a51fe15d59244f90, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733949419672 2024-12-11T20:37:02,487 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:02,487 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#B#compaction#491 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:02,488 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/a9030e2b1c7e4fbeb96aed4cdc696636 is 50, key is test_row_0/B:col10/1733949419672/Put/seqid=0 2024-12-11T20:37:02,494 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211ffa3ac3521f7481ba924cd2a9656ef3d_c0305045e5dcee8aa8d5a9ea3acd16e6 store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:02,495 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211ffa3ac3521f7481ba924cd2a9656ef3d_c0305045e5dcee8aa8d5a9ea3acd16e6, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:02,495 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211ffa3ac3521f7481ba924cd2a9656ef3d_c0305045e5dcee8aa8d5a9ea3acd16e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:02,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742404_1580 (size=12459) 2024-12-11T20:37:02,503 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/a9030e2b1c7e4fbeb96aed4cdc696636 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/a9030e2b1c7e4fbeb96aed4cdc696636 2024-12-11T20:37:02,508 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/B of c0305045e5dcee8aa8d5a9ea3acd16e6 into a9030e2b1c7e4fbeb96aed4cdc696636(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:02,508 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:02,508 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/B, priority=13, startTime=1733949422480; duration=0sec 2024-12-11T20:37:02,508 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:37:02,508 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:B 2024-12-11T20:37:02,508 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:37:02,509 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:37:02,509 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/C is initiating minor compaction (all files) 2024-12-11T20:37:02,509 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/C in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,509 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/be0fa8b9c90841968f3631e25bcade8b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f03ce7e164ea42ef8aa4d1b3c3ba04b1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/75d1206cc9ab417e9361242fcaa1ff9b] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=35.5 K 2024-12-11T20:37:02,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742405_1581 (size=4469) 2024-12-11T20:37:02,509 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting be0fa8b9c90841968f3631e25bcade8b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733949417847 2024-12-11T20:37:02,510 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f03ce7e164ea42ef8aa4d1b3c3ba04b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733949418490 2024-12-11T20:37:02,510 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 75d1206cc9ab417e9361242fcaa1ff9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733949419672 2024-12-11T20:37:02,511 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#A#compaction#492 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:02,512 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/8d23213689694ff08db8749e7beeb37f is 175, key is test_row_0/A:col10/1733949419672/Put/seqid=0 2024-12-11T20:37:02,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742406_1582 (size=31413) 2024-12-11T20:37:02,518 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#C#compaction#493 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:02,519 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/fc4fc3b17ff44b10886e0222183d41df is 50, key is test_row_0/C:col10/1733949419672/Put/seqid=0 2024-12-11T20:37:02,524 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/8d23213689694ff08db8749e7beeb37f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/8d23213689694ff08db8749e7beeb37f 2024-12-11T20:37:02,528 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/A of c0305045e5dcee8aa8d5a9ea3acd16e6 into 8d23213689694ff08db8749e7beeb37f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:02,528 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:02,528 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/A, priority=13, startTime=1733949422480; duration=0sec 2024-12-11T20:37:02,528 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:02,528 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:A 2024-12-11T20:37:02,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742407_1583 (size=12459) 2024-12-11T20:37:02,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-11T20:37:02,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:02,625 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T20:37:02,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:37:02,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:02,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:37:02,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:02,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:37:02,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:02,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d5d51007fb704e9289bc2dd8fbc0b13b_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949420856/Put/seqid=0 2024-12-11T20:37:02,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742408_1584 (size=12304) 2024-12-11T20:37:02,934 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/fc4fc3b17ff44b10886e0222183d41df as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/fc4fc3b17ff44b10886e0222183d41df 2024-12-11T20:37:02,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T20:37:02,937 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/C of c0305045e5dcee8aa8d5a9ea3acd16e6 into fc4fc3b17ff44b10886e0222183d41df(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:02,937 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:02,937 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/C, priority=13, startTime=1733949422480; duration=0sec 2024-12-11T20:37:02,937 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:02,937 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:C 2024-12-11T20:37:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:03,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:03,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:03,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949483032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,035 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,037 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211d5d51007fb704e9289bc2dd8fbc0b13b_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d5d51007fb704e9289bc2dd8fbc0b13b_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:03,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5c99c86c82c64d559620875c9730ad60, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:03,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949483034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949483035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5c99c86c82c64d559620875c9730ad60 is 175, key is test_row_0/A:col10/1733949420856/Put/seqid=0 2024-12-11T20:37:03,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742409_1585 (size=31105) 2024-12-11T20:37:03,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949483146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949483147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949483147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949483356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949483356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949483357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,442 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5c99c86c82c64d559620875c9730ad60 2024-12-11T20:37:03,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/21149f2d6fb549beafe3cd9605e4ff99 is 50, key is test_row_0/B:col10/1733949420856/Put/seqid=0 2024-12-11T20:37:03,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742410_1586 (size=12151) 2024-12-11T20:37:03,460 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/21149f2d6fb549beafe3cd9605e4ff99 2024-12-11T20:37:03,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/b6f2c6d7a3644c67b13ddbeb0974a3f2 is 50, key is test_row_0/C:col10/1733949420856/Put/seqid=0 2024-12-11T20:37:03,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742411_1587 (size=12151) 2024-12-11T20:37:03,477 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/b6f2c6d7a3644c67b13ddbeb0974a3f2 2024-12-11T20:37:03,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5c99c86c82c64d559620875c9730ad60 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5c99c86c82c64d559620875c9730ad60 2024-12-11T20:37:03,484 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5c99c86c82c64d559620875c9730ad60, entries=150, sequenceid=157, filesize=30.4 K 2024-12-11T20:37:03,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/21149f2d6fb549beafe3cd9605e4ff99 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/21149f2d6fb549beafe3cd9605e4ff99 2024-12-11T20:37:03,487 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/21149f2d6fb549beafe3cd9605e4ff99, entries=150, sequenceid=157, filesize=11.9 K 2024-12-11T20:37:03,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/b6f2c6d7a3644c67b13ddbeb0974a3f2 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/b6f2c6d7a3644c67b13ddbeb0974a3f2 2024-12-11T20:37:03,491 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/b6f2c6d7a3644c67b13ddbeb0974a3f2, entries=150, sequenceid=157, filesize=11.9 K 2024-12-11T20:37:03,491 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 866ms, sequenceid=157, compaction requested=false 2024-12-11T20:37:03,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:03,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:03,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-11T20:37:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-11T20:37:03,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-11T20:37:03,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7020 sec 2024-12-11T20:37:03,495 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.7060 sec 2024-12-11T20:37:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:03,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-11T20:37:03,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:37:03,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:03,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:37:03,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:03,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:37:03,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:03,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412119b6bb6f73d594df9b1efd38302548755_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949423667/Put/seqid=0 2024-12-11T20:37:03,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742412_1588 (size=12304) 2024-12-11T20:37:03,692 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:03,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949483692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,695 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412119b6bb6f73d594df9b1efd38302548755_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412119b6bb6f73d594df9b1efd38302548755_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:03,696 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/c383a8401aae425bbeae306f716f70eb, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:03,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/c383a8401aae425bbeae306f716f70eb is 175, key is test_row_0/A:col10/1733949423667/Put/seqid=0 2024-12-11T20:37:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949483694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949483698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742413_1589 (size=31105) 2024-12-11T20:37:03,709 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/c383a8401aae425bbeae306f716f70eb 2024-12-11T20:37:03,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/3076a5202bde4901a7a9fd8f0b92dce5 is 50, key is test_row_0/B:col10/1733949423667/Put/seqid=0 2024-12-11T20:37:03,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742414_1590 (size=12151) 2024-12-11T20:37:03,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/3076a5202bde4901a7a9fd8f0b92dce5 2024-12-11T20:37:03,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/9239e143b5fc4563b3f74e74037471ac is 50, key is test_row_0/C:col10/1733949423667/Put/seqid=0 2024-12-11T20:37:03,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742415_1591 (size=12151) 2024-12-11T20:37:03,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949483807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949483807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:03,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:03,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949483811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:03,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949484016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949484016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949484016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/9239e143b5fc4563b3f74e74037471ac 2024-12-11T20:37:04,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/c383a8401aae425bbeae306f716f70eb as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/c383a8401aae425bbeae306f716f70eb 2024-12-11T20:37:04,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/c383a8401aae425bbeae306f716f70eb, entries=150, sequenceid=175, filesize=30.4 K 2024-12-11T20:37:04,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/3076a5202bde4901a7a9fd8f0b92dce5 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/3076a5202bde4901a7a9fd8f0b92dce5 2024-12-11T20:37:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/3076a5202bde4901a7a9fd8f0b92dce5, entries=150, sequenceid=175, filesize=11.9 K 2024-12-11T20:37:04,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/9239e143b5fc4563b3f74e74037471ac as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/9239e143b5fc4563b3f74e74037471ac 2024-12-11T20:37:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/9239e143b5fc4563b3f74e74037471ac, entries=150, sequenceid=175, filesize=11.9 K 2024-12-11T20:37:04,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 603ms, sequenceid=175, compaction requested=true 2024-12-11T20:37:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:37:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:04,272 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:37:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:37:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:04,272 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:37:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:37:04,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:37:04,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,279 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:37:04,279 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/A is initiating minor compaction (all files) 2024-12-11T20:37:04,279 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/A in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,279 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/8d23213689694ff08db8749e7beeb37f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5c99c86c82c64d559620875c9730ad60, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/c383a8401aae425bbeae306f716f70eb] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=91.4 K 2024-12-11T20:37:04,279 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:04,279 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/8d23213689694ff08db8749e7beeb37f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5c99c86c82c64d559620875c9730ad60, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/c383a8401aae425bbeae306f716f70eb] 2024-12-11T20:37:04,279 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:37:04,279 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/B is initiating minor compaction (all files) 2024-12-11T20:37:04,279 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/B in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:04,280 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/a9030e2b1c7e4fbeb96aed4cdc696636, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/21149f2d6fb549beafe3cd9605e4ff99, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/3076a5202bde4901a7a9fd8f0b92dce5] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=35.9 K 2024-12-11T20:37:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,280 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting a9030e2b1c7e4fbeb96aed4cdc696636, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733949419672 2024-12-11T20:37:04,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,280 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d23213689694ff08db8749e7beeb37f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733949419672 2024-12-11T20:37:04,280 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 21149f2d6fb549beafe3cd9605e4ff99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733949420856 2024-12-11T20:37:04,281 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c99c86c82c64d559620875c9730ad60, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733949420856 2024-12-11T20:37:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,282 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 3076a5202bde4901a7a9fd8f0b92dce5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733949423033 2024-12-11T20:37:04,282 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting c383a8401aae425bbeae306f716f70eb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733949423033 2024-12-11T20:37:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,293 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#B#compaction#500 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:04,293 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/d24431553a004c15927ea545160b8c39 is 50, key is test_row_0/B:col10/1733949423667/Put/seqid=0 2024-12-11T20:37:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,300 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:04,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,307 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412116690263a2e8d41f08b4dd953316e7044_c0305045e5dcee8aa8d5a9ea3acd16e6 store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,316 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412116690263a2e8d41f08b4dd953316e7044_c0305045e5dcee8aa8d5a9ea3acd16e6, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:04,316 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412116690263a2e8d41f08b4dd953316e7044_c0305045e5dcee8aa8d5a9ea3acd16e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742417_1593 (size=4469) 2024-12-11T20:37:04,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742416_1592 (size=12561) 2024-12-11T20:37:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,326 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/d24431553a004c15927ea545160b8c39 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/d24431553a004c15927ea545160b8c39 2024-12-11T20:37:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,330 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/B of c0305045e5dcee8aa8d5a9ea3acd16e6 into d24431553a004c15927ea545160b8c39(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:04,330 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:04,330 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/B, priority=13, startTime=1733949424272; duration=0sec 2024-12-11T20:37:04,330 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:37:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:04,330 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:B 2024-12-11T20:37:04,330 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:37:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,330 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-11T20:37:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,331 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:37:04,331 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/C is initiating minor compaction (all files) 2024-12-11T20:37:04,331 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/C in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,331 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/fc4fc3b17ff44b10886e0222183d41df, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/b6f2c6d7a3644c67b13ddbeb0974a3f2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/9239e143b5fc4563b3f74e74037471ac] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=35.9 K 2024-12-11T20:37:04,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:37:04,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:04,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:37:04,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:04,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:37:04,332 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:04,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,332 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting fc4fc3b17ff44b10886e0222183d41df, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1733949419672 2024-12-11T20:37:04,332 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting b6f2c6d7a3644c67b13ddbeb0974a3f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733949420856 2024-12-11T20:37:04,333 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 9239e143b5fc4563b3f74e74037471ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733949423033 2024-12-11T20:37:04,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211ba4540da95e641448b2cf0f541f0cf5b_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949424329/Put/seqid=0 2024-12-11T20:37:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742418_1594 (size=14794) 2024-12-11T20:37:04,353 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:04,353 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/f954ca42169e42b4b832e3815238eaef is 50, key is test_row_0/C:col10/1733949423667/Put/seqid=0 2024-12-11T20:37:04,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742419_1595 (size=12561) 2024-12-11T20:37:04,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949484357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949484358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949484361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,364 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949484467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949484467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949484467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949484676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949484677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949484677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,721 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:04,721 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/e98294e5a311449c9f097e107b1ef7bb is 175, key is test_row_0/A:col10/1733949423667/Put/seqid=0 2024-12-11T20:37:04,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742420_1596 (size=31515) 2024-12-11T20:37:04,753 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:04,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211ba4540da95e641448b2cf0f541f0cf5b_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211ba4540da95e641448b2cf0f541f0cf5b_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:04,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/e2eefceee2394e92a1f0cccbf9265627, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:04,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/e2eefceee2394e92a1f0cccbf9265627 is 175, key is test_row_0/A:col10/1733949424329/Put/seqid=0 2024-12-11T20:37:04,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742421_1597 (size=39745) 2024-12-11T20:37:04,763 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/f954ca42169e42b4b832e3815238eaef as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f954ca42169e42b4b832e3815238eaef 2024-12-11T20:37:04,767 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/C of c0305045e5dcee8aa8d5a9ea3acd16e6 into f954ca42169e42b4b832e3815238eaef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:04,767 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:04,767 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/C, priority=13, startTime=1733949424272; duration=0sec 2024-12-11T20:37:04,767 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:04,767 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:C 2024-12-11T20:37:04,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-11T20:37:04,957 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:37:04,957 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:37:04,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-11T20:37:04,959 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:37:04,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T20:37:04,959 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:37:04,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:37:04,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949484987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949484987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949484987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T20:37:05,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-11T20:37:05,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:05,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:05,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:05,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,127 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/e98294e5a311449c9f097e107b1ef7bb as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e98294e5a311449c9f097e107b1ef7bb 2024-12-11T20:37:05,130 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/A of c0305045e5dcee8aa8d5a9ea3acd16e6 into e98294e5a311449c9f097e107b1ef7bb(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:05,130 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:05,130 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/A, priority=13, startTime=1733949424272; duration=0sec 2024-12-11T20:37:05,130 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:05,130 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:A 2024-12-11T20:37:05,160 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/e2eefceee2394e92a1f0cccbf9265627 2024-12-11T20:37:05,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/8f901752dcb946be91742847889c9d32 is 50, key is test_row_0/B:col10/1733949424329/Put/seqid=0 2024-12-11T20:37:05,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742422_1598 (size=9757) 2024-12-11T20:37:05,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-11T20:37:05,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:05,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:05,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:05,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T20:37:05,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-11T20:37:05,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:05,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:05,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:05,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949485497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949485496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:05,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949485497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:05,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:05,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:05,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/8f901752dcb946be91742847889c9d32 2024-12-11T20:37:05,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-11T20:37:05,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:05,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:05,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:05,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:05,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/8849ab4935014b0a8181fd7168412db3 is 50, key is test_row_0/C:col10/1733949424329/Put/seqid=0 2024-12-11T20:37:05,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742423_1599 (size=9757) 2024-12-11T20:37:05,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/8849ab4935014b0a8181fd7168412db3 2024-12-11T20:37:05,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/e2eefceee2394e92a1f0cccbf9265627 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e2eefceee2394e92a1f0cccbf9265627 2024-12-11T20:37:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T20:37:05,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e2eefceee2394e92a1f0cccbf9265627, entries=200, sequenceid=193, filesize=38.8 K 2024-12-11T20:37:05,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/8f901752dcb946be91742847889c9d32 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/8f901752dcb946be91742847889c9d32 2024-12-11T20:37:05,595 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/8f901752dcb946be91742847889c9d32, entries=100, sequenceid=193, filesize=9.5 K 2024-12-11T20:37:05,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/8849ab4935014b0a8181fd7168412db3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/8849ab4935014b0a8181fd7168412db3 2024-12-11T20:37:05,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/8849ab4935014b0a8181fd7168412db3, entries=100, sequenceid=193, filesize=9.5 K 2024-12-11T20:37:05,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1271ms, sequenceid=193, compaction requested=false 2024-12-11T20:37:05,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:05,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-11T20:37:05,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:05,722 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-11T20:37:05,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:37:05,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:05,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:37:05,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:05,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:37:05,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:05,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bf67caf808504927b2b7480afa0b0be7_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949424357/Put/seqid=0 2024-12-11T20:37:05,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742424_1600 (size=12304) 2024-12-11T20:37:05,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:05,733 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211bf67caf808504927b2b7480afa0b0be7_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bf67caf808504927b2b7480afa0b0be7_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:05,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/04dba5a01e8a45aca2e78e574c5aed2f, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:05,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/04dba5a01e8a45aca2e78e574c5aed2f is 175, key is test_row_0/A:col10/1733949424357/Put/seqid=0 2024-12-11T20:37:05,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742425_1601 (size=31105) 2024-12-11T20:37:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T20:37:06,138 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/04dba5a01e8a45aca2e78e574c5aed2f 2024-12-11T20:37:06,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/6cdca01b29284c00848043ce5570cbbe is 50, key is test_row_0/B:col10/1733949424357/Put/seqid=0 2024-12-11T20:37:06,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742426_1602 (size=12151) 2024-12-11T20:37:06,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:06,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:06,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:06,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949486525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:06,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:06,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949486526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:06,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949486528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:06,547 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/6cdca01b29284c00848043ce5570cbbe 2024-12-11T20:37:06,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/6e0c56c57d774228bd86bb3df0088671 is 50, key is test_row_0/C:col10/1733949424357/Put/seqid=0 2024-12-11T20:37:06,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742427_1603 (size=12151) 2024-12-11T20:37:06,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949486637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949486637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:06,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949486637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:06,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949486846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:06,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949486847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:06,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949486847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:06,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:06,954 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/6e0c56c57d774228bd86bb3df0088671 2024-12-11T20:37:06,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/04dba5a01e8a45aca2e78e574c5aed2f as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/04dba5a01e8a45aca2e78e574c5aed2f 2024-12-11T20:37:06,959 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/04dba5a01e8a45aca2e78e574c5aed2f, entries=150, sequenceid=210, filesize=30.4 K 2024-12-11T20:37:06,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/6cdca01b29284c00848043ce5570cbbe as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/6cdca01b29284c00848043ce5570cbbe 2024-12-11T20:37:06,962 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/6cdca01b29284c00848043ce5570cbbe, entries=150, sequenceid=210, filesize=11.9 K 2024-12-11T20:37:06,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/6e0c56c57d774228bd86bb3df0088671 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/6e0c56c57d774228bd86bb3df0088671 2024-12-11T20:37:06,966 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/6e0c56c57d774228bd86bb3df0088671, entries=150, sequenceid=210, filesize=11.9 K 2024-12-11T20:37:06,967 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1244ms, sequenceid=210, compaction requested=true 2024-12-11T20:37:06,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:06,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:06,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-11T20:37:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-11T20:37:06,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-11T20:37:06,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0090 sec 2024-12-11T20:37:06,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.0130 sec 2024-12-11T20:37:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-11T20:37:07,107 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:37:07,107 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:37:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-11T20:37:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-11T20:37:07,109 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:37:07,109 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:37:07,109 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:37:07,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:07,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-11T20:37:07,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:37:07,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:07,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:37:07,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:07,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:37:07,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:07,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121123991f5c11f84542b2b52500e98341a6_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949427160/Put/seqid=0 2024-12-11T20:37:07,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742428_1604 (size=17284) 2024-12-11T20:37:07,173 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-11T20:37:07,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949487173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949487174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949487175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,178 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-11T20:37:07,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-11T20:37:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:07,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949487286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949487286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949487287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-11T20:37:07,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:07,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:07,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:07,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-11T20:37:07,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949487496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949487496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949487496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,499 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-11T20:37:07,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:07,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:07,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:07,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,571 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:07,574 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121123991f5c11f84542b2b52500e98341a6_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121123991f5c11f84542b2b52500e98341a6_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:07,574 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/73b4475a81ad4d679e10a91719fb71f4, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:07,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/73b4475a81ad4d679e10a91719fb71f4 is 175, key is test_row_0/A:col10/1733949427160/Put/seqid=0 2024-12-11T20:37:07,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742429_1605 (size=48389) 2024-12-11T20:37:07,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-11T20:37:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:07,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-11T20:37:07,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949487807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949487807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949487807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:07,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:07,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:07,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-11T20:37:07,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:07,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:07,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:07,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:07,978 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/73b4475a81ad4d679e10a91719fb71f4 2024-12-11T20:37:07,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/1853dd39047e434caa58ebab80556552 is 50, key is test_row_0/B:col10/1733949427160/Put/seqid=0 2024-12-11T20:37:07,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742430_1606 (size=12151) 2024-12-11T20:37:07,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/1853dd39047e434caa58ebab80556552 2024-12-11T20:37:07,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/25cdf51536884b1ea15e9f66b081c919 is 50, key is test_row_0/C:col10/1733949427160/Put/seqid=0 2024-12-11T20:37:08,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742431_1607 (size=12151) 2024-12-11T20:37:08,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-11T20:37:08,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:08,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:08,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:08,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:08,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:08,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-11T20:37:08,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:08,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:08,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:08,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:08,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-11T20:37:08,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:08,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949488317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:08,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949488317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:08,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949488317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:08,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:08,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-11T20:37:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:08,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:08,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:08,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:08,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/25cdf51536884b1ea15e9f66b081c919 2024-12-11T20:37:08,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/73b4475a81ad4d679e10a91719fb71f4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/73b4475a81ad4d679e10a91719fb71f4 2024-12-11T20:37:08,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/73b4475a81ad4d679e10a91719fb71f4, entries=250, sequenceid=230, filesize=47.3 K 2024-12-11T20:37:08,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/1853dd39047e434caa58ebab80556552 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/1853dd39047e434caa58ebab80556552 2024-12-11T20:37:08,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/1853dd39047e434caa58ebab80556552, entries=150, sequenceid=230, filesize=11.9 K 2024-12-11T20:37:08,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/25cdf51536884b1ea15e9f66b081c919 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/25cdf51536884b1ea15e9f66b081c919 2024-12-11T20:37:08,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/25cdf51536884b1ea15e9f66b081c919, entries=150, sequenceid=230, filesize=11.9 K 2024-12-11T20:37:08,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1253ms, sequenceid=230, compaction requested=true 2024-12-11T20:37:08,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:08,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:37:08,413 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:37:08,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:08,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:37:08,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:08,413 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:37:08,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:37:08,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:37:08,414 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150754 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:37:08,414 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:37:08,414 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/A is initiating minor compaction (all files) 2024-12-11T20:37:08,414 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/B is initiating minor compaction (all files) 2024-12-11T20:37:08,414 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/A in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:08,414 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/B in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:08,414 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e98294e5a311449c9f097e107b1ef7bb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e2eefceee2394e92a1f0cccbf9265627, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/04dba5a01e8a45aca2e78e574c5aed2f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/73b4475a81ad4d679e10a91719fb71f4] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=147.2 K 2024-12-11T20:37:08,414 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/d24431553a004c15927ea545160b8c39, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/8f901752dcb946be91742847889c9d32, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/6cdca01b29284c00848043ce5570cbbe, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/1853dd39047e434caa58ebab80556552] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=45.5 K 2024-12-11T20:37:08,414 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:08,414 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e98294e5a311449c9f097e107b1ef7bb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e2eefceee2394e92a1f0cccbf9265627, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/04dba5a01e8a45aca2e78e574c5aed2f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/73b4475a81ad4d679e10a91719fb71f4] 2024-12-11T20:37:08,414 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting e98294e5a311449c9f097e107b1ef7bb, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733949423033 2024-12-11T20:37:08,414 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting d24431553a004c15927ea545160b8c39, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733949423033 2024-12-11T20:37:08,415 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2eefceee2394e92a1f0cccbf9265627, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733949423690 2024-12-11T20:37:08,415 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f901752dcb946be91742847889c9d32, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733949424329 2024-12-11T20:37:08,415 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04dba5a01e8a45aca2e78e574c5aed2f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733949424354 2024-12-11T20:37:08,415 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73b4475a81ad4d679e10a91719fb71f4, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733949426523 2024-12-11T20:37:08,415 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cdca01b29284c00848043ce5570cbbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733949424354 2024-12-11T20:37:08,415 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 1853dd39047e434caa58ebab80556552, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733949426523 2024-12-11T20:37:08,420 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:08,421 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211c3398e67f4ba44a99db37e274f885a7b_c0305045e5dcee8aa8d5a9ea3acd16e6 store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:08,423 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211c3398e67f4ba44a99db37e274f885a7b_c0305045e5dcee8aa8d5a9ea3acd16e6, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:08,423 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211c3398e67f4ba44a99db37e274f885a7b_c0305045e5dcee8aa8d5a9ea3acd16e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:08,423 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#B#compaction#513 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:08,424 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/b3608004876d417787af1bae45bf5646 is 50, key is test_row_0/B:col10/1733949427160/Put/seqid=0 2024-12-11T20:37:08,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742432_1608 (size=4469) 2024-12-11T20:37:08,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742433_1609 (size=12697) 2024-12-11T20:37:08,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-11T20:37:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:08,482 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-11T20:37:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:37:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:37:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:37:08,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:08,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211fa958fe25482434ca3b48594484d9822_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949427165/Put/seqid=0 2024-12-11T20:37:08,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742434_1610 (size=12304) 2024-12-11T20:37:08,827 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#A#compaction#512 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:08,828 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/9b0c81ff26ed4e9787ad073341e3aac1 is 175, key is test_row_0/A:col10/1733949427160/Put/seqid=0 2024-12-11T20:37:08,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742435_1611 (size=31651) 2024-12-11T20:37:08,843 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/b3608004876d417787af1bae45bf5646 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/b3608004876d417787af1bae45bf5646 2024-12-11T20:37:08,847 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/B of c0305045e5dcee8aa8d5a9ea3acd16e6 into b3608004876d417787af1bae45bf5646(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:08,847 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:08,847 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/B, priority=12, startTime=1733949428413; duration=0sec 2024-12-11T20:37:08,847 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:37:08,847 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:B 2024-12-11T20:37:08,847 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-11T20:37:08,848 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-11T20:37:08,848 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/C is initiating minor compaction (all files) 2024-12-11T20:37:08,848 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/C in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:08,848 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f954ca42169e42b4b832e3815238eaef, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/8849ab4935014b0a8181fd7168412db3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/6e0c56c57d774228bd86bb3df0088671, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/25cdf51536884b1ea15e9f66b081c919] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=45.5 K 2024-12-11T20:37:08,850 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f954ca42169e42b4b832e3815238eaef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1733949423033 2024-12-11T20:37:08,851 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 8849ab4935014b0a8181fd7168412db3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733949424329 2024-12-11T20:37:08,851 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e0c56c57d774228bd86bb3df0088671, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733949424354 2024-12-11T20:37:08,851 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 25cdf51536884b1ea15e9f66b081c919, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733949426523 2024-12-11T20:37:08,859 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#C#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:08,860 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/4f515293a3264267bf37c0ea0448c654 is 50, key is test_row_0/C:col10/1733949427160/Put/seqid=0 2024-12-11T20:37:08,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742436_1612 (size=12697) 2024-12-11T20:37:08,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:08,897 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211fa958fe25482434ca3b48594484d9822_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211fa958fe25482434ca3b48594484d9822_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:08,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/099c96254c4848fe9341904035c88ac3, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:08,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/099c96254c4848fe9341904035c88ac3 is 175, key is test_row_0/A:col10/1733949427165/Put/seqid=0 2024-12-11T20:37:08,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742437_1613 (size=31105) 2024-12-11T20:37:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-11T20:37:09,290 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/9b0c81ff26ed4e9787ad073341e3aac1 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9b0c81ff26ed4e9787ad073341e3aac1 2024-12-11T20:37:09,290 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/4f515293a3264267bf37c0ea0448c654 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/4f515293a3264267bf37c0ea0448c654 2024-12-11T20:37:09,297 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/A of c0305045e5dcee8aa8d5a9ea3acd16e6 into 9b0c81ff26ed4e9787ad073341e3aac1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:09,297 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/C of c0305045e5dcee8aa8d5a9ea3acd16e6 into 4f515293a3264267bf37c0ea0448c654(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:09,297 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:09,297 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:09,297 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/A, priority=12, startTime=1733949428413; duration=0sec 2024-12-11T20:37:09,297 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/C, priority=12, startTime=1733949428413; duration=0sec 2024-12-11T20:37:09,298 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:09,298 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:A 2024-12-11T20:37:09,298 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:09,298 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:C 2024-12-11T20:37:09,302 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/099c96254c4848fe9341904035c88ac3 2024-12-11T20:37:09,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/b6551d0202b340ed8b2714711356a817 is 50, key is test_row_0/B:col10/1733949427165/Put/seqid=0 2024-12-11T20:37:09,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742438_1614 (size=12151) 2024-12-11T20:37:09,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:09,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:09,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949489350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:09,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949489353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,355 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:09,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949489354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:09,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949489457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949489457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:09,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949489467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:09,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949489667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949489667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:09,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949489677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:09,715 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/b6551d0202b340ed8b2714711356a817 2024-12-11T20:37:09,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/53cb0c44036b4dba8887bc01c05a7cb4 is 50, key is test_row_0/C:col10/1733949427165/Put/seqid=0 2024-12-11T20:37:09,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742439_1615 (size=12151) 2024-12-11T20:37:09,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949489976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949489976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:09,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:09,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949489987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:09,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,123 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/53cb0c44036b4dba8887bc01c05a7cb4 2024-12-11T20:37:10,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/099c96254c4848fe9341904035c88ac3 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/099c96254c4848fe9341904035c88ac3 2024-12-11T20:37:10,129 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/099c96254c4848fe9341904035c88ac3, entries=150, sequenceid=243, filesize=30.4 K 2024-12-11T20:37:10,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/b6551d0202b340ed8b2714711356a817 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/b6551d0202b340ed8b2714711356a817 2024-12-11T20:37:10,132 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/b6551d0202b340ed8b2714711356a817, entries=150, sequenceid=243, filesize=11.9 K 2024-12-11T20:37:10,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/53cb0c44036b4dba8887bc01c05a7cb4 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/53cb0c44036b4dba8887bc01c05a7cb4 2024-12-11T20:37:10,135 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/53cb0c44036b4dba8887bc01c05a7cb4, entries=150, sequenceid=243, filesize=11.9 K 2024-12-11T20:37:10,136 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1654ms, sequenceid=243, compaction requested=false 2024-12-11T20:37:10,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:10,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:10,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-11T20:37:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-11T20:37:10,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-11T20:37:10,139 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0280 sec 2024-12-11T20:37:10,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.0320 sec 2024-12-11T20:37:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:10,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-11T20:37:10,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:37:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:37:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:37:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:37:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:37:10,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-11T20:37:10,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121125ecec2f79cb4020bfa0868a9a0bbe54_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949429346/Put/seqid=0 2024-12-11T20:37:10,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742440_1616 (size=14994) 2024-12-11T20:37:10,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949490501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949490504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949490504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57640 deadline: 1733949490607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:10,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57624 deadline: 1733949490607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:10,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949490607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:10,609 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0305045e5dcee8aa8d5a9ea3acd16e6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:37:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,609 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5be53b084ac7,44543,1733949248752 for 'test_row_2' in c0305045e5dcee8aa8d5a9ea3acd16e6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-11T20:37:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949490617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949490617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949490817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949490827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:10,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949490827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:10,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:10,898 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:10,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024121125ecec2f79cb4020bfa0868a9a0bbe54_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121125ecec2f79cb4020bfa0868a9a0bbe54_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:10,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5d1aa2e2f607413abd3957d625ea8466, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:10,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5d1aa2e2f607413abd3957d625ea8466 is 175, key is test_row_0/A:col10/1733949429346/Put/seqid=0 2024-12-11T20:37:10,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742441_1617 (size=39949) 2024-12-11T20:37:11,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949491126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:11,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:11,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:11,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949491136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:11,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:11,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:11,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949491137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:11,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-11T20:37:11,277 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:37:11,277 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-11T20:37:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-11T20:37:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-11T20:37:11,280 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-11T20:37:11,280 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-11T20:37:11,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-11T20:37:11,305 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5d1aa2e2f607413abd3957d625ea8466 2024-12-11T20:37:11,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/f99b236f12ef4f90b3b8d2c658c037e0 is 50, key is test_row_0/B:col10/1733949429346/Put/seqid=0 2024-12-11T20:37:11,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742442_1618 (size=12301) 2024-12-11T20:37:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-11T20:37:11,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-11T20:37:11,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:11,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:11,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:11,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-11T20:37:11,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:11,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:11,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:11,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-11T20:37:11,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57630 deadline: 1733949491637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:11,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57572 deadline: 1733949491647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-11T20:37:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44543 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57570 deadline: 1733949491647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 , the old value is region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-11T20:37:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., hostname=5be53b084ac7,44543,1733949248752, seqNum=5 because the exception is null or not the one we care about 2024-12-11T20:37:11,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/f99b236f12ef4f90b3b8d2c658c037e0 2024-12-11T20:37:11,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/f8c598988a3840faaf2775cea6d03e36 is 50, key is test_row_0/C:col10/1733949429346/Put/seqid=0 2024-12-11T20:37:11,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742443_1619 (size=12301) 2024-12-11T20:37:11,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-11T20:37:11,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:11,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:11,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:11,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-11T20:37:11,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:11,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:11,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:11,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-11T20:37:12,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-11T20:37:12,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:12,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:12,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:12,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:12,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:12,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-11T20:37:12,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/f8c598988a3840faaf2775cea6d03e36 2024-12-11T20:37:12,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/5d1aa2e2f607413abd3957d625ea8466 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5d1aa2e2f607413abd3957d625ea8466 2024-12-11T20:37:12,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5d1aa2e2f607413abd3957d625ea8466, entries=200, sequenceid=267, filesize=39.0 K 2024-12-11T20:37:12,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/f99b236f12ef4f90b3b8d2c658c037e0 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/f99b236f12ef4f90b3b8d2c658c037e0 2024-12-11T20:37:12,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/f99b236f12ef4f90b3b8d2c658c037e0, entries=150, sequenceid=267, filesize=12.0 K 2024-12-11T20:37:12,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/f8c598988a3840faaf2775cea6d03e36 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f8c598988a3840faaf2775cea6d03e36 2024-12-11T20:37:12,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f8c598988a3840faaf2775cea6d03e36, entries=150, sequenceid=267, filesize=12.0 K 2024-12-11T20:37:12,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1650ms, sequenceid=267, compaction requested=true 2024-12-11T20:37:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:A, priority=-2147483648, current under compaction store size is 1 2024-12-11T20:37:12,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:12,138 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:37:12,139 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:37:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:B, priority=-2147483648, current under compaction store size is 2 2024-12-11T20:37:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c0305045e5dcee8aa8d5a9ea3acd16e6:C, priority=-2147483648, current under compaction store size is 3 2024-12-11T20:37:12,139 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:37:12,140 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:37:12,140 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:37:12,140 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/A is initiating minor compaction (all files) 2024-12-11T20:37:12,140 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/B is initiating minor compaction (all files) 2024-12-11T20:37:12,140 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/A in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:12,140 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/B in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:12,140 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9b0c81ff26ed4e9787ad073341e3aac1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/099c96254c4848fe9341904035c88ac3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5d1aa2e2f607413abd3957d625ea8466] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=100.3 K 2024-12-11T20:37:12,140 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/b3608004876d417787af1bae45bf5646, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/b6551d0202b340ed8b2714711356a817, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/f99b236f12ef4f90b3b8d2c658c037e0] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=36.3 K 2024-12-11T20:37:12,140 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:12,140 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. files: [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9b0c81ff26ed4e9787ad073341e3aac1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/099c96254c4848fe9341904035c88ac3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5d1aa2e2f607413abd3957d625ea8466] 2024-12-11T20:37:12,141 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting b3608004876d417787af1bae45bf5646, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733949426523 2024-12-11T20:37:12,141 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b0c81ff26ed4e9787ad073341e3aac1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733949426523 2024-12-11T20:37:12,142 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting b6551d0202b340ed8b2714711356a817, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733949427165 2024-12-11T20:37:12,142 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f99b236f12ef4f90b3b8d2c658c037e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733949429346 2024-12-11T20:37:12,142 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 099c96254c4848fe9341904035c88ac3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733949427165 2024-12-11T20:37:12,142 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d1aa2e2f607413abd3957d625ea8466, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733949429346 2024-12-11T20:37:12,147 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#B#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:12,148 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/6953ca0fb58f4cd5a5267bf12eea8b62 is 50, key is test_row_0/B:col10/1733949429346/Put/seqid=0 2024-12-11T20:37:12,151 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:12,175 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241211f9a9d40c250242dabfee171785bd0b8a_c0305045e5dcee8aa8d5a9ea3acd16e6 store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:12,177 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241211f9a9d40c250242dabfee171785bd0b8a_c0305045e5dcee8aa8d5a9ea3acd16e6, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:12,177 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211f9a9d40c250242dabfee171785bd0b8a_c0305045e5dcee8aa8d5a9ea3acd16e6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:12,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742444_1620 (size=12949) 2024-12-11T20:37:12,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742445_1621 (size=4469) 2024-12-11T20:37:12,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44543 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-11T20:37:12,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:12,195 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-11T20:37:12,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:37:12,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:12,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:37:12,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:12,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:37:12,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:12,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211dc860cd8a5d0425485e0b1338fdde266_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949430503/Put/seqid=0 2024-12-11T20:37:12,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742446_1622 (size=12454) 2024-12-11T20:37:12,326 INFO [Thread-2444 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2444. 2024-12-11T20:37:12,326 DEBUG [Thread-2444 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:37:12,326 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,326 DEBUG [Thread-2444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:12,327 INFO [Thread-2438 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2438. 2024-12-11T20:37:12,327 DEBUG [Thread-2438 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:37:12,327 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,327 DEBUG [Thread-2438 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:12,327 INFO [Thread-2446 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2446. 2024-12-11T20:37:12,328 DEBUG [Thread-2446 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:37:12,328 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,328 DEBUG [Thread-2446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:12,328 INFO [Thread-2442 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2442. 2024-12-11T20:37:12,328 DEBUG [Thread-2442 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:37:12,328 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,328 DEBUG [Thread-2442 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,328 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:12,329 INFO [Thread-2440 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2440. 2024-12-11T20:37:12,329 DEBUG [Thread-2440 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:37:12,329 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,329 DEBUG [Thread-2440 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,329 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-11T20:37:12,583 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#A#compaction#522 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:12,584 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/1006754b41ea448a9405baa426bb8d52 is 175, key is test_row_0/A:col10/1733949429346/Put/seqid=0 2024-12-11T20:37:12,588 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/6953ca0fb58f4cd5a5267bf12eea8b62 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/6953ca0fb58f4cd5a5267bf12eea8b62 2024-12-11T20:37:12,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742447_1623 (size=31903) 2024-12-11T20:37:12,594 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/B of c0305045e5dcee8aa8d5a9ea3acd16e6 into 6953ca0fb58f4cd5a5267bf12eea8b62(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:12,594 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:12,594 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/B, priority=13, startTime=1733949432138; duration=0sec 2024-12-11T20:37:12,594 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-11T20:37:12,594 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:B 2024-12-11T20:37:12,594 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-11T20:37:12,595 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-11T20:37:12,595 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1541): c0305045e5dcee8aa8d5a9ea3acd16e6/C is initiating minor compaction (all files) 2024-12-11T20:37:12,595 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c0305045e5dcee8aa8d5a9ea3acd16e6/C in TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:12,595 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/4f515293a3264267bf37c0ea0448c654, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/53cb0c44036b4dba8887bc01c05a7cb4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f8c598988a3840faaf2775cea6d03e36] into tmpdir=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp, totalSize=36.3 K 2024-12-11T20:37:12,596 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f515293a3264267bf37c0ea0448c654, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1733949426523 2024-12-11T20:37:12,596 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting 53cb0c44036b4dba8887bc01c05a7cb4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1733949427165 2024-12-11T20:37:12,596 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] compactions.Compactor(225): Compacting f8c598988a3840faaf2775cea6d03e36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733949429346 2024-12-11T20:37:12,601 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c0305045e5dcee8aa8d5a9ea3acd16e6#C#compaction#524 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-11T20:37:12,602 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/62985cd824294fada49b9031c82bf381 is 50, key is test_row_0/C:col10/1733949429346/Put/seqid=0 2024-12-11T20:37:12,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742448_1624 (size=12949) 2024-12-11T20:37:12,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:12,612 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241211dc860cd8a5d0425485e0b1338fdde266_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211dc860cd8a5d0425485e0b1338fdde266_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:12,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/92d90370879c41f2a748831fd0970103, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:12,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/92d90370879c41f2a748831fd0970103 is 175, key is test_row_0/A:col10/1733949430503/Put/seqid=0 2024-12-11T20:37:12,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742449_1625 (size=31255) 2024-12-11T20:37:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44543 {}] regionserver.HRegion(8855): Flush requested on c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:12,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. as already flushing 2024-12-11T20:37:12,648 INFO [Thread-2431 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2431. 2024-12-11T20:37:12,648 DEBUG [Thread-2431 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:37:12,648 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,648 DEBUG [Thread-2431 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:12,659 INFO [Thread-2427 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2427. 2024-12-11T20:37:12,659 INFO [Thread-2433 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2433. 2024-12-11T20:37:12,659 DEBUG [Thread-2433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:37:12,659 DEBUG [Thread-2427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:37:12,659 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,659 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,659 DEBUG [Thread-2427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,659 DEBUG [Thread-2433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:12,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:12,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:12,996 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/1006754b41ea448a9405baa426bb8d52 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/1006754b41ea448a9405baa426bb8d52 2024-12-11T20:37:13,001 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/A of c0305045e5dcee8aa8d5a9ea3acd16e6 into 1006754b41ea448a9405baa426bb8d52(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:13,002 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:13,002 INFO [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/A, priority=13, startTime=1733949432138; duration=0sec 2024-12-11T20:37:13,002 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:13,002 DEBUG [RS:0;5be53b084ac7:44543-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:A 2024-12-11T20:37:13,011 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/62985cd824294fada49b9031c82bf381 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/62985cd824294fada49b9031c82bf381 2024-12-11T20:37:13,016 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c0305045e5dcee8aa8d5a9ea3acd16e6/C of c0305045e5dcee8aa8d5a9ea3acd16e6 into 62985cd824294fada49b9031c82bf381(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-11T20:37:13,016 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:13,016 INFO [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., storeName=c0305045e5dcee8aa8d5a9ea3acd16e6/C, priority=13, startTime=1733949432139; duration=0sec 2024-12-11T20:37:13,016 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-11T20:37:13,016 DEBUG [RS:0;5be53b084ac7:44543-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c0305045e5dcee8aa8d5a9ea3acd16e6:C 2024-12-11T20:37:13,016 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/92d90370879c41f2a748831fd0970103 2024-12-11T20:37:13,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/46923d6dd1ea4b7ea2d30f1f8751df8b is 50, key is test_row_0/B:col10/1733949430503/Put/seqid=0 2024-12-11T20:37:13,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742450_1626 (size=12301) 2024-12-11T20:37:13,427 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/46923d6dd1ea4b7ea2d30f1f8751df8b 2024-12-11T20:37:13,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/176f40d37835404eb3623b624b7618ae is 50, key is test_row_0/C:col10/1733949430503/Put/seqid=0 2024-12-11T20:37:13,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742451_1627 (size=12301) 2024-12-11T20:37:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-11T20:37:13,836 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/176f40d37835404eb3623b624b7618ae 2024-12-11T20:37:13,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/92d90370879c41f2a748831fd0970103 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/92d90370879c41f2a748831fd0970103 2024-12-11T20:37:13,844 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/92d90370879c41f2a748831fd0970103, entries=150, sequenceid=279, filesize=30.5 K 2024-12-11T20:37:13,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/46923d6dd1ea4b7ea2d30f1f8751df8b as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/46923d6dd1ea4b7ea2d30f1f8751df8b 2024-12-11T20:37:13,848 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/46923d6dd1ea4b7ea2d30f1f8751df8b, entries=150, sequenceid=279, filesize=12.0 K 2024-12-11T20:37:13,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/176f40d37835404eb3623b624b7618ae as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/176f40d37835404eb3623b624b7618ae 2024-12-11T20:37:13,851 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/176f40d37835404eb3623b624b7618ae, entries=150, sequenceid=279, filesize=12.0 K 2024-12-11T20:37:13,852 INFO [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1657ms, sequenceid=279, compaction requested=false 2024-12-11T20:37:13,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for c0305045e5dcee8aa8d5a9ea3acd16e6: 2024-12-11T20:37:13,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:13,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5be53b084ac7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-11T20:37:13,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-11T20:37:13,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-11T20:37:13,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5730 sec 2024-12-11T20:37:13,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.5780 sec 2024-12-11T20:37:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-11T20:37:15,448 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:37:20,662 INFO [Thread-2429 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2429. 2024-12-11T20:37:20,662 DEBUG [Thread-2429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:37:20,662 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:20,662 DEBUG [Thread-2429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:20,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:20,711 INFO [Thread-2435 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2435. 2024-12-11T20:37:20,712 DEBUG [Thread-2435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-11T20:37:20,712 DEBUG [Thread-2435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:20,712 DEBUG [Thread-2435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:20,712 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4824 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4609 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4937 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4827 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4628 2024-12-11T20:37:20,713 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-11T20:37:20,713 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T20:37:20,714 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:37:20,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:20,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:20,714 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:20,715 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-11T20:37:20,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-11T20:37:20,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-11T20:37:20,719 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949440718"}]},"ts":"1733949440718"} 2024-12-11T20:37:20,720 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-11T20:37:20,720 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-11T20:37:20,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-11T20:37:20,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, UNASSIGN}] 2024-12-11T20:37:20,723 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, UNASSIGN 2024-12-11T20:37:20,724 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=c0305045e5dcee8aa8d5a9ea3acd16e6, regionState=CLOSING, regionLocation=5be53b084ac7,44543,1733949248752 2024-12-11T20:37:20,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, UNASSIGN because future has completed 2024-12-11T20:37:20,726 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-11T20:37:20,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752}] 2024-12-11T20:37:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-11T20:37:20,880 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:20,880 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-11T20:37:20,880 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing c0305045e5dcee8aa8d5a9ea3acd16e6, disabling compactions & flushes 2024-12-11T20:37:20,880 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:20,881 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:20,881 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. after waiting 0 ms 2024-12-11T20:37:20,881 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:20,881 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing c0305045e5dcee8aa8d5a9ea3acd16e6 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-11T20:37:20,882 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=A 2024-12-11T20:37:20,882 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:20,882 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=B 2024-12-11T20:37:20,882 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:20,882 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c0305045e5dcee8aa8d5a9ea3acd16e6, store=C 2024-12-11T20:37:20,882 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-11T20:37:20,891 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412112bb045a3efbb4452a272d452bec0cd34_c0305045e5dcee8aa8d5a9ea3acd16e6 is 50, key is test_row_0/A:col10/1733949432657/Put/seqid=0 2024-12-11T20:37:20,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742452_1628 (size=9914) 2024-12-11T20:37:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-11T20:37:21,296 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-11T20:37:21,303 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412112bb045a3efbb4452a272d452bec0cd34_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412112bb045a3efbb4452a272d452bec0cd34_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:21,304 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/3f6b955a948744eeb210f01e5720fd62, store: [table=TestAcidGuarantees family=A region=c0305045e5dcee8aa8d5a9ea3acd16e6] 2024-12-11T20:37:21,305 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/3f6b955a948744eeb210f01e5720fd62 is 175, key is test_row_0/A:col10/1733949432657/Put/seqid=0 2024-12-11T20:37:21,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742453_1629 (size=22561) 2024-12-11T20:37:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-11T20:37:21,711 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/3f6b955a948744eeb210f01e5720fd62 2024-12-11T20:37:21,725 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/152a35eb7d1344e6a7989e2988bd0fc8 is 50, key is test_row_0/B:col10/1733949432657/Put/seqid=0 2024-12-11T20:37:21,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742454_1630 (size=9857) 2024-12-11T20:37:21,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-11T20:37:22,130 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/152a35eb7d1344e6a7989e2988bd0fc8 2024-12-11T20:37:22,143 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/05ad668e13fc49a79783ae201093e528 is 50, key is test_row_0/C:col10/1733949432657/Put/seqid=0 2024-12-11T20:37:22,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742455_1631 (size=9857) 2024-12-11T20:37:22,548 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/05ad668e13fc49a79783ae201093e528 2024-12-11T20:37:22,556 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/A/3f6b955a948744eeb210f01e5720fd62 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/3f6b955a948744eeb210f01e5720fd62 2024-12-11T20:37:22,563 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/3f6b955a948744eeb210f01e5720fd62, entries=100, sequenceid=290, filesize=22.0 K 2024-12-11T20:37:22,564 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/B/152a35eb7d1344e6a7989e2988bd0fc8 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/152a35eb7d1344e6a7989e2988bd0fc8 2024-12-11T20:37:22,567 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/152a35eb7d1344e6a7989e2988bd0fc8, entries=100, sequenceid=290, filesize=9.6 K 2024-12-11T20:37:22,568 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/.tmp/C/05ad668e13fc49a79783ae201093e528 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/05ad668e13fc49a79783ae201093e528 2024-12-11T20:37:22,571 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/05ad668e13fc49a79783ae201093e528, entries=100, sequenceid=290, filesize=9.6 K 2024-12-11T20:37:22,572 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1691ms, sequenceid=290, compaction requested=true 2024-12-11T20:37:22,573 DEBUG [StoreCloser-TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/a58d8621f42c41d9ad5e659e285f3e05, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9d395d55578547b7a527e502e7b2223b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b2f9f4ab8ad54084a053d094f8c2cc47, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b5498fafe5c442409881b17bc26996f0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/95fccb9c293c4e03936191359c622e6a, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/cb97c8d4f9614957b0e7d23ec16cc016, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/03d548a357b3449ba11d041e680e7e5f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e7d68b5ea7584fbd85fc155fd3fe19f1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/8d23213689694ff08db8749e7beeb37f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5098f7ed372a4f02a51fe15d59244f90, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5c99c86c82c64d559620875c9730ad60, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e98294e5a311449c9f097e107b1ef7bb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/c383a8401aae425bbeae306f716f70eb, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e2eefceee2394e92a1f0cccbf9265627, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/04dba5a01e8a45aca2e78e574c5aed2f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/73b4475a81ad4d679e10a91719fb71f4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9b0c81ff26ed4e9787ad073341e3aac1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/099c96254c4848fe9341904035c88ac3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5d1aa2e2f607413abd3957d625ea8466] to archive 2024-12-11T20:37:22,574 DEBUG [StoreCloser-TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:37:22,576 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b5498fafe5c442409881b17bc26996f0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b5498fafe5c442409881b17bc26996f0 2024-12-11T20:37:22,576 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b2f9f4ab8ad54084a053d094f8c2cc47 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/b2f9f4ab8ad54084a053d094f8c2cc47 2024-12-11T20:37:22,576 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/cb97c8d4f9614957b0e7d23ec16cc016 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/cb97c8d4f9614957b0e7d23ec16cc016 2024-12-11T20:37:22,577 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/95fccb9c293c4e03936191359c622e6a to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/95fccb9c293c4e03936191359c622e6a 2024-12-11T20:37:22,577 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9d395d55578547b7a527e502e7b2223b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9d395d55578547b7a527e502e7b2223b 2024-12-11T20:37:22,577 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/a58d8621f42c41d9ad5e659e285f3e05 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/a58d8621f42c41d9ad5e659e285f3e05 2024-12-11T20:37:22,577 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e7d68b5ea7584fbd85fc155fd3fe19f1 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e7d68b5ea7584fbd85fc155fd3fe19f1 2024-12-11T20:37:22,577 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/03d548a357b3449ba11d041e680e7e5f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/03d548a357b3449ba11d041e680e7e5f 2024-12-11T20:37:22,578 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5c99c86c82c64d559620875c9730ad60 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5c99c86c82c64d559620875c9730ad60 2024-12-11T20:37:22,578 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/8d23213689694ff08db8749e7beeb37f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/8d23213689694ff08db8749e7beeb37f 2024-12-11T20:37:22,578 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5098f7ed372a4f02a51fe15d59244f90 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5098f7ed372a4f02a51fe15d59244f90 2024-12-11T20:37:22,579 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/c383a8401aae425bbeae306f716f70eb to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/c383a8401aae425bbeae306f716f70eb 2024-12-11T20:37:22,579 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e2eefceee2394e92a1f0cccbf9265627 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e2eefceee2394e92a1f0cccbf9265627 2024-12-11T20:37:22,579 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/04dba5a01e8a45aca2e78e574c5aed2f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/04dba5a01e8a45aca2e78e574c5aed2f 2024-12-11T20:37:22,579 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/73b4475a81ad4d679e10a91719fb71f4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/73b4475a81ad4d679e10a91719fb71f4 2024-12-11T20:37:22,579 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9b0c81ff26ed4e9787ad073341e3aac1 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/9b0c81ff26ed4e9787ad073341e3aac1 2024-12-11T20:37:22,579 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5d1aa2e2f607413abd3957d625ea8466 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/5d1aa2e2f607413abd3957d625ea8466 2024-12-11T20:37:22,579 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/099c96254c4848fe9341904035c88ac3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/099c96254c4848fe9341904035c88ac3 2024-12-11T20:37:22,583 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e98294e5a311449c9f097e107b1ef7bb to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/e98294e5a311449c9f097e107b1ef7bb 2024-12-11T20:37:22,584 DEBUG [StoreCloser-TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/152a07c17b144a2bbab4d058d85ccded, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/359bfb05480c4dd588ff93b2ada6fd0f, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/2cbb8b36d0e247b0affc54134b79276b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/ba991c077a544f0b954d18439459cf71, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/590d06ed92e94df388491ac6121272cf, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/9fb344c43801487d8d3f1b8d9cd9ba17, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/64d9fd8c738d41678a1526ca92a3e2ce, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/629de55302334dc69bda53bf900ad1d0, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/a9030e2b1c7e4fbeb96aed4cdc696636, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/640e85a0a4294cc0b30b6fbb29cf84d6, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/21149f2d6fb549beafe3cd9605e4ff99, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/d24431553a004c15927ea545160b8c39, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/3076a5202bde4901a7a9fd8f0b92dce5, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/8f901752dcb946be91742847889c9d32, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/6cdca01b29284c00848043ce5570cbbe, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/b3608004876d417787af1bae45bf5646, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/1853dd39047e434caa58ebab80556552, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/b6551d0202b340ed8b2714711356a817, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/f99b236f12ef4f90b3b8d2c658c037e0] to archive 2024-12-11T20:37:22,585 DEBUG [StoreCloser-TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:37:22,586 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/64d9fd8c738d41678a1526ca92a3e2ce to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/64d9fd8c738d41678a1526ca92a3e2ce 2024-12-11T20:37:22,586 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/2cbb8b36d0e247b0affc54134b79276b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/2cbb8b36d0e247b0affc54134b79276b 2024-12-11T20:37:22,586 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/152a07c17b144a2bbab4d058d85ccded to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/152a07c17b144a2bbab4d058d85ccded 2024-12-11T20:37:22,586 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/ba991c077a544f0b954d18439459cf71 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/ba991c077a544f0b954d18439459cf71 2024-12-11T20:37:22,586 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/629de55302334dc69bda53bf900ad1d0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/629de55302334dc69bda53bf900ad1d0 2024-12-11T20:37:22,586 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/590d06ed92e94df388491ac6121272cf to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/590d06ed92e94df388491ac6121272cf 2024-12-11T20:37:22,587 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/9fb344c43801487d8d3f1b8d9cd9ba17 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/9fb344c43801487d8d3f1b8d9cd9ba17 2024-12-11T20:37:22,587 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/359bfb05480c4dd588ff93b2ada6fd0f to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/359bfb05480c4dd588ff93b2ada6fd0f 2024-12-11T20:37:22,587 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/640e85a0a4294cc0b30b6fbb29cf84d6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/640e85a0a4294cc0b30b6fbb29cf84d6 2024-12-11T20:37:22,588 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/a9030e2b1c7e4fbeb96aed4cdc696636 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/a9030e2b1c7e4fbeb96aed4cdc696636 2024-12-11T20:37:22,588 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/d24431553a004c15927ea545160b8c39 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/d24431553a004c15927ea545160b8c39 2024-12-11T20:37:22,588 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/3076a5202bde4901a7a9fd8f0b92dce5 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/3076a5202bde4901a7a9fd8f0b92dce5 2024-12-11T20:37:22,588 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/8f901752dcb946be91742847889c9d32 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/8f901752dcb946be91742847889c9d32 2024-12-11T20:37:22,588 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/21149f2d6fb549beafe3cd9605e4ff99 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/21149f2d6fb549beafe3cd9605e4ff99 2024-12-11T20:37:22,588 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/6cdca01b29284c00848043ce5570cbbe to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/6cdca01b29284c00848043ce5570cbbe 2024-12-11T20:37:22,588 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/b3608004876d417787af1bae45bf5646 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/b3608004876d417787af1bae45bf5646 2024-12-11T20:37:22,588 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/1853dd39047e434caa58ebab80556552 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/1853dd39047e434caa58ebab80556552 2024-12-11T20:37:22,588 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/b6551d0202b340ed8b2714711356a817 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/b6551d0202b340ed8b2714711356a817 2024-12-11T20:37:22,590 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/f99b236f12ef4f90b3b8d2c658c037e0 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/f99b236f12ef4f90b3b8d2c658c037e0 2024-12-11T20:37:22,591 DEBUG [StoreCloser-TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/5defc68ec7c04599894c22b8b6a31d4c, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/38b376a2ddee493bb980156faae3d480, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/8c5b155b8ef64acda7d7453a1cb55da9, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/c690f38492274381a1bc496d43f04ca9, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/e22ab6a407e94903acbe58a3adfb3910, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/be0fa8b9c90841968f3631e25bcade8b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/4957cb75921742d3959398c8426fc63d, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f03ce7e164ea42ef8aa4d1b3c3ba04b1, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/fc4fc3b17ff44b10886e0222183d41df, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/75d1206cc9ab417e9361242fcaa1ff9b, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/b6f2c6d7a3644c67b13ddbeb0974a3f2, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f954ca42169e42b4b832e3815238eaef, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/9239e143b5fc4563b3f74e74037471ac, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/8849ab4935014b0a8181fd7168412db3, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/6e0c56c57d774228bd86bb3df0088671, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/4f515293a3264267bf37c0ea0448c654, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/25cdf51536884b1ea15e9f66b081c919, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/53cb0c44036b4dba8887bc01c05a7cb4, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f8c598988a3840faaf2775cea6d03e36] to archive 2024-12-11T20:37:22,592 DEBUG [StoreCloser-TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.-1 {}] backup.HFileArchiver(363): Archiving compacted files. 2024-12-11T20:37:22,593 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/e22ab6a407e94903acbe58a3adfb3910 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/e22ab6a407e94903acbe58a3adfb3910 2024-12-11T20:37:22,593 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/4957cb75921742d3959398c8426fc63d to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/4957cb75921742d3959398c8426fc63d 2024-12-11T20:37:22,593 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/5defc68ec7c04599894c22b8b6a31d4c to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/5defc68ec7c04599894c22b8b6a31d4c 2024-12-11T20:37:22,594 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/c690f38492274381a1bc496d43f04ca9 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/c690f38492274381a1bc496d43f04ca9 2024-12-11T20:37:22,594 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/be0fa8b9c90841968f3631e25bcade8b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/be0fa8b9c90841968f3631e25bcade8b 2024-12-11T20:37:22,594 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/38b376a2ddee493bb980156faae3d480 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/38b376a2ddee493bb980156faae3d480 2024-12-11T20:37:22,594 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f03ce7e164ea42ef8aa4d1b3c3ba04b1 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f03ce7e164ea42ef8aa4d1b3c3ba04b1 2024-12-11T20:37:22,594 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/8c5b155b8ef64acda7d7453a1cb55da9 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/8c5b155b8ef64acda7d7453a1cb55da9 2024-12-11T20:37:22,595 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/fc4fc3b17ff44b10886e0222183d41df to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/fc4fc3b17ff44b10886e0222183d41df 2024-12-11T20:37:22,595 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/75d1206cc9ab417e9361242fcaa1ff9b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/75d1206cc9ab417e9361242fcaa1ff9b 2024-12-11T20:37:22,595 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/b6f2c6d7a3644c67b13ddbeb0974a3f2 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/b6f2c6d7a3644c67b13ddbeb0974a3f2 2024-12-11T20:37:22,595 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/9239e143b5fc4563b3f74e74037471ac to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/9239e143b5fc4563b3f74e74037471ac 2024-12-11T20:37:22,595 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f954ca42169e42b4b832e3815238eaef to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f954ca42169e42b4b832e3815238eaef 2024-12-11T20:37:22,595 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/8849ab4935014b0a8181fd7168412db3 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/8849ab4935014b0a8181fd7168412db3 2024-12-11T20:37:22,595 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/6e0c56c57d774228bd86bb3df0088671 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/6e0c56c57d774228bd86bb3df0088671 2024-12-11T20:37:22,595 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/4f515293a3264267bf37c0ea0448c654 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/4f515293a3264267bf37c0ea0448c654 2024-12-11T20:37:22,596 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/25cdf51536884b1ea15e9f66b081c919 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/25cdf51536884b1ea15e9f66b081c919 2024-12-11T20:37:22,596 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/53cb0c44036b4dba8887bc01c05a7cb4 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/53cb0c44036b4dba8887bc01c05a7cb4 2024-12-11T20:37:22,596 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileableStoreFile, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f8c598988a3840faaf2775cea6d03e36 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/f8c598988a3840faaf2775cea6d03e36 2024-12-11T20:37:22,599 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/recovered.edits/293.seqid, newMaxSeqId=293, maxSeqId=4 2024-12-11T20:37:22,599 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. 2024-12-11T20:37:22,599 DEBUG [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for c0305045e5dcee8aa8d5a9ea3acd16e6: Waiting for close lock at 1733949440880Running coprocessor pre-close hooks at 1733949440880Disabling compacts and flushes for region at 1733949440880Disabling writes for close at 1733949440881 (+1 ms)Obtaining lock to block concurrent updates at 1733949440881Preparing flush snapshotting stores in c0305045e5dcee8aa8d5a9ea3acd16e6 at 1733949440881Finished memstore snapshotting TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733949440882 (+1 ms)Flushing stores of TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6. at 1733949440884 (+2 ms)Flushing c0305045e5dcee8aa8d5a9ea3acd16e6/A: creating writer at 1733949440884Flushing mob file c0305045e5dcee8aa8d5a9ea3acd16e6/A: appending metadata at 1733949440890 (+6 ms)Flushing mob file c0305045e5dcee8aa8d5a9ea3acd16e6/A: closing flushed file at 1733949440890Flushing c0305045e5dcee8aa8d5a9ea3acd16e6/A: appending metadata at 1733949441304 (+414 ms)Flushing c0305045e5dcee8aa8d5a9ea3acd16e6/A: closing flushed file at 1733949441305 (+1 ms)Flushing c0305045e5dcee8aa8d5a9ea3acd16e6/B: creating writer at 1733949441720 (+415 ms)Flushing c0305045e5dcee8aa8d5a9ea3acd16e6/B: appending metadata at 1733949441724 (+4 ms)Flushing c0305045e5dcee8aa8d5a9ea3acd16e6/B: closing flushed file at 1733949441724Flushing c0305045e5dcee8aa8d5a9ea3acd16e6/C: creating writer at 1733949442138 (+414 ms)Flushing c0305045e5dcee8aa8d5a9ea3acd16e6/C: appending metadata at 1733949442142 (+4 ms)Flushing c0305045e5dcee8aa8d5a9ea3acd16e6/C: closing flushed file at 1733949442142Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f01a44: reopening flushed file at 1733949442555 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bb9c67d: reopening flushed file at 1733949442563 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@512d86a7: reopening flushed file at 1733949442567 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c0305045e5dcee8aa8d5a9ea3acd16e6 in 1691ms, sequenceid=290, compaction requested=true at 1733949442572 (+5 ms)Writing region close event to WAL at 1733949442597 (+25 ms)Running coprocessor post-close hooks at 1733949442599 (+2 ms)Closed at 1733949442599 2024-12-11T20:37:22,601 INFO [RS_CLOSE_REGION-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,601 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=c0305045e5dcee8aa8d5a9ea3acd16e6, regionState=CLOSED 2024-12-11T20:37:22,602 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 because future has completed 2024-12-11T20:37:22,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-11T20:37:22,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure c0305045e5dcee8aa8d5a9ea3acd16e6, server=5be53b084ac7,44543,1733949248752 in 1.8770 sec 2024-12-11T20:37:22,605 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-12-11T20:37:22,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c0305045e5dcee8aa8d5a9ea3acd16e6, UNASSIGN in 1.8820 sec 2024-12-11T20:37:22,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-11T20:37:22,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8850 sec 2024-12-11T20:37:22,608 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733949442608"}]},"ts":"1733949442608"} 2024-12-11T20:37:22,609 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-11T20:37:22,609 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-11T20:37:22,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8950 sec 2024-12-11T20:37:22,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-11T20:37:22,868 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:37:22,868 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-11T20:37:22,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:37:22,871 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:37:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-11T20:37:22,872 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:37:22,875 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,879 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C, FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/recovered.edits] 2024-12-11T20:37:22,884 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/1006754b41ea448a9405baa426bb8d52 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/1006754b41ea448a9405baa426bb8d52 2024-12-11T20:37:22,884 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/3f6b955a948744eeb210f01e5720fd62 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/3f6b955a948744eeb210f01e5720fd62 2024-12-11T20:37:22,885 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/92d90370879c41f2a748831fd0970103 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/A/92d90370879c41f2a748831fd0970103 2024-12-11T20:37:22,889 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/152a35eb7d1344e6a7989e2988bd0fc8 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/152a35eb7d1344e6a7989e2988bd0fc8 2024-12-11T20:37:22,889 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/46923d6dd1ea4b7ea2d30f1f8751df8b to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/46923d6dd1ea4b7ea2d30f1f8751df8b 2024-12-11T20:37:22,889 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/6953ca0fb58f4cd5a5267bf12eea8b62 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/B/6953ca0fb58f4cd5a5267bf12eea8b62 2024-12-11T20:37:22,893 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/05ad668e13fc49a79783ae201093e528 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/05ad668e13fc49a79783ae201093e528 2024-12-11T20:37:22,894 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/176f40d37835404eb3623b624b7618ae to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/176f40d37835404eb3623b624b7618ae 2024-12-11T20:37:22,894 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/62985cd824294fada49b9031c82bf381 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/C/62985cd824294fada49b9031c82bf381 2024-12-11T20:37:22,898 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/recovered.edits/293.seqid to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6/recovered.edits/293.seqid 2024-12-11T20:37:22,899 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/default/TestAcidGuarantees/c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,899 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-11T20:37:22,899 DEBUG [PEWorker-4 {}] backup.HFileArchiver(133): ARCHIVING hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T20:37:22,900 DEBUG [PEWorker-4 {}] backup.HFileArchiver(161): Archiving [FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-11T20:37:22,909 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121123991f5c11f84542b2b52500e98341a6_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121123991f5c11f84542b2b52500e98341a6_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,909 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121105618ff3e8d444088918fde761e5a901_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121105618ff3e8d444088918fde761e5a901_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,910 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211248d4a30b12d493eb68264aae177bbd1_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211248d4a30b12d493eb68264aae177bbd1_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,910 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121125ecec2f79cb4020bfa0868a9a0bbe54_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121125ecec2f79cb4020bfa0868a9a0bbe54_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,910 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115e92ae7eba2d4a3db34673454d1e5713_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412115e92ae7eba2d4a3db34673454d1e5713_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,910 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117afb9eb7fb02459e93c06a0922105f44_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117afb9eb7fb02459e93c06a0922105f44_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,910 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412112bb045a3efbb4452a272d452bec0cd34_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412112bb045a3efbb4452a272d452bec0cd34_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,911 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117e09c5c3fefa4d1e8d1e1df51fcd2878_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412117e09c5c3fefa4d1e8d1e1df51fcd2878_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,912 DEBUG [HFileArchiver-34 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412119b6bb6f73d594df9b1efd38302548755_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412119b6bb6f73d594df9b1efd38302548755_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,912 DEBUG [HFileArchiver-36 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211ba4540da95e641448b2cf0f541f0cf5b_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211ba4540da95e641448b2cf0f541f0cf5b_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,912 DEBUG [HFileArchiver-39 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211dc860cd8a5d0425485e0b1338fdde266_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211dc860cd8a5d0425485e0b1338fdde266_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,912 DEBUG [HFileArchiver-37 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bf67caf808504927b2b7480afa0b0be7_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211bf67caf808504927b2b7480afa0b0be7_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,913 DEBUG [HFileArchiver-38 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d5d51007fb704e9289bc2dd8fbc0b13b_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211d5d51007fb704e9289bc2dd8fbc0b13b_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,913 DEBUG [HFileArchiver-35 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e2802a1aee4f42a797c044375fe5aef0_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211e2802a1aee4f42a797c044375fe5aef0_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,913 DEBUG [HFileArchiver-33 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121187551799cd044b8386a5a40c8c4c7c42_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024121187551799cd044b8386a5a40c8c4c7c42_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,913 DEBUG [HFileArchiver-40 {}] backup.HFileArchiver(620): Archived from FileablePath, hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211fa958fe25482434ca3b48594484d9822_c0305045e5dcee8aa8d5a9ea3acd16e6 to hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241211fa958fe25482434ca3b48594484d9822_c0305045e5dcee8aa8d5a9ea3acd16e6 2024-12-11T20:37:22,914 DEBUG [PEWorker-4 {}] backup.HFileArchiver(634): Deleted hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-11T20:37:22,916 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:37:22,919 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(371): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-11T20:37:22,921 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(408): Removing 'TestAcidGuarantees' descriptor. 2024-12-11T20:37:22,922 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:37:22,922 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(398): Removing 'TestAcidGuarantees' from region states. 2024-12-11T20:37:22,923 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733949442922"}]},"ts":"9223372036854775807"} 2024-12-11T20:37:22,924 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-11T20:37:22,924 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c0305045e5dcee8aa8d5a9ea3acd16e6, NAME => 'TestAcidGuarantees,,1733949409183.c0305045e5dcee8aa8d5a9ea3acd16e6.', STARTKEY => '', ENDKEY => ''}] 2024-12-11T20:37:22,924 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(402): Marking 'TestAcidGuarantees' as deleted. 2024-12-11T20:37:22,924 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733949442924"}]},"ts":"9223372036854775807"} 2024-12-11T20:37:22,926 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-11T20:37:22,926 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-11T20:37:22,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 58 msec 2024-12-11T20:37:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39923 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-11T20:37:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-11T20:37:22,977 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-11T20:37:22,992 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=255 (was 256), OpenFileDescriptor=443 (was 443), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=394 (was 352) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1830 (was 1857) 2024-12-11T20:37:22,992 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-11T20:37:22,992 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T20:37:22,992 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:37:22,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:22,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:22,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-11T20:37:22,992 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-11T20:37:22,992 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=688211475, stopped=false 2024-12-11T20:37:22,993 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5be53b084ac7,39923,1733949248041 2024-12-11T20:37:23,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T20:37:23,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-11T20:37:23,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:37:23,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:37:23,049 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T20:37:23,050 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-11T20:37:23,050 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:37:23,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:23,051 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5be53b084ac7,44543,1733949248752' ***** 2024-12-11T20:37:23,051 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-11T20:37:23,051 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T20:37:23,051 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-11T20:37:23,052 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.HeapMemoryManager(224): Stopping 2024-12-11T20:37:23,052 INFO [RS:0;5be53b084ac7:44543 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-11T20:37:23,052 INFO [RS:0;5be53b084ac7:44543 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-11T20:37:23,053 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(959): stopping server 5be53b084ac7,44543,1733949248752 2024-12-11T20:37:23,053 INFO [RS:0;5be53b084ac7:44543 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T20:37:23,053 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-11T20:37:23,053 INFO [RS:0;5be53b084ac7:44543 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5be53b084ac7:44543. 2024-12-11T20:37:23,053 DEBUG [RS:0;5be53b084ac7:44543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-11T20:37:23,053 DEBUG [RS:0;5be53b084ac7:44543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:23,053 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-11T20:37:23,053 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-11T20:37:23,053 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-11T20:37:23,053 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-11T20:37:23,054 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-11T20:37:23,054 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-11T20:37:23,055 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T20:37:23,055 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-11T20:37:23,055 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-11T20:37:23,055 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-11T20:37:23,055 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-11T20:37:23,055 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-11T20:37:23,055 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-11T20:37:23,078 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/info/2ffe4e28407249919d7bcfd9ec44df31 is 95, key is TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f./info:/1733949280287/DeleteFamily/seqid=0 2024-12-11T20:37:23,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742456_1632 (size=6514) 2024-12-11T20:37:23,115 INFO [regionserver/5be53b084ac7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T20:37:23,255 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T20:37:23,455 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T20:37:23,482 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/info/2ffe4e28407249919d7bcfd9ec44df31 2024-12-11T20:37:23,498 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/ns/58a6c31a2b104eed86ae67edba7f9214 is 93, key is TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f./ns:/1733949280228/DeleteFamily/seqid=0 2024-12-11T20:37:23,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742457_1633 (size=5906) 2024-12-11T20:37:23,656 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T20:37:23,845 INFO [regionserver/5be53b084ac7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-11T20:37:23,846 INFO [regionserver/5be53b084ac7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-11T20:37:23,856 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T20:37:23,902 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/ns/58a6c31a2b104eed86ae67edba7f9214 2024-12-11T20:37:23,925 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/rep_barrier/00ff47cc128746e4864732236008cf68 is 102, key is TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f./rep_barrier:/1733949280228/DeleteFamily/seqid=0 2024-12-11T20:37:23,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742458_1634 (size=6025) 2024-12-11T20:37:24,056 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T20:37:24,257 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T20:37:24,329 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/rep_barrier/00ff47cc128746e4864732236008cf68 2024-12-11T20:37:24,353 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/table/479b86139022412981cb050e89c25d9e is 96, key is TestAcidGuarantees,,1733949253386.c0a19b15b87adee237235cecb1e4c99f./table:/1733949280228/DeleteFamily/seqid=0 2024-12-11T20:37:24,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742459_1635 (size=5936) 2024-12-11T20:37:24,457 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T20:37:24,658 DEBUG [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-11T20:37:24,757 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/table/479b86139022412981cb050e89c25d9e 2024-12-11T20:37:24,766 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/info/2ffe4e28407249919d7bcfd9ec44df31 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/info/2ffe4e28407249919d7bcfd9ec44df31 2024-12-11T20:37:24,770 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/info/2ffe4e28407249919d7bcfd9ec44df31, entries=12, sequenceid=90, filesize=6.4 K 2024-12-11T20:37:24,771 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/ns/58a6c31a2b104eed86ae67edba7f9214 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/ns/58a6c31a2b104eed86ae67edba7f9214 2024-12-11T20:37:24,775 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/ns/58a6c31a2b104eed86ae67edba7f9214, entries=8, sequenceid=90, filesize=5.8 K 2024-12-11T20:37:24,776 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/rep_barrier/00ff47cc128746e4864732236008cf68 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/rep_barrier/00ff47cc128746e4864732236008cf68 2024-12-11T20:37:24,779 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/rep_barrier/00ff47cc128746e4864732236008cf68, entries=6, sequenceid=90, filesize=5.9 K 2024-12-11T20:37:24,780 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/.tmp/table/479b86139022412981cb050e89c25d9e as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/table/479b86139022412981cb050e89c25d9e 2024-12-11T20:37:24,784 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/table/479b86139022412981cb050e89c25d9e, entries=7, sequenceid=90, filesize=5.8 K 2024-12-11T20:37:24,784 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1729ms, sequenceid=90, compaction requested=false 2024-12-11T20:37:24,788 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-11T20:37:24,788 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-11T20:37:24,789 INFO [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-11T20:37:24,789 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733949443054Running coprocessor pre-close hooks at 1733949443055 (+1 ms)Disabling compacts and flushes for region at 1733949443055Disabling writes for close at 1733949443055Obtaining lock to block concurrent updates at 1733949443055Preparing flush snapshotting stores in 1588230740 at 1733949443055Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733949443056 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733949443057 (+1 ms)Flushing 1588230740/info: creating writer at 1733949443058 (+1 ms)Flushing 1588230740/info: appending metadata at 1733949443077 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733949443077Flushing 1588230740/ns: creating writer at 1733949443485 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733949443497 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733949443497Flushing 1588230740/rep_barrier: creating writer at 1733949443909 (+412 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733949443924 (+15 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733949443925 (+1 ms)Flushing 1588230740/table: creating writer at 1733949444336 (+411 ms)Flushing 1588230740/table: appending metadata at 1733949444352 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733949444352Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a07f7ff: reopening flushed file at 1733949444765 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@457fd353: reopening flushed file at 1733949444771 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6adf34b2: reopening flushed file at 1733949444775 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50178f1b: reopening flushed file at 1733949444779 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1729ms, sequenceid=90, compaction requested=false at 1733949444784 (+5 ms)Writing region close event to WAL at 1733949444786 (+2 ms)Running coprocessor post-close hooks at 1733949444788 (+2 ms)Closed at 1733949444789 (+1 ms) 2024-12-11T20:37:24,789 DEBUG [RS_CLOSE_META-regionserver/5be53b084ac7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-11T20:37:24,858 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(976): stopping server 5be53b084ac7,44543,1733949248752; all regions closed. 2024-12-11T20:37:24,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741834_1010 (size=25344) 2024-12-11T20:37:24,864 DEBUG [RS:0;5be53b084ac7:44543 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/oldWALs 2024-12-11T20:37:24,864 INFO [RS:0;5be53b084ac7:44543 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5be53b084ac7%2C44543%2C1733949248752.meta:.meta(num 1733949252908) 2024-12-11T20:37:24,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741832_1008 (size=16485942) 2024-12-11T20:37:24,868 DEBUG [RS:0;5be53b084ac7:44543 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/oldWALs 2024-12-11T20:37:24,868 INFO [RS:0;5be53b084ac7:44543 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5be53b084ac7%2C44543%2C1733949248752:(num 1733949251928) 2024-12-11T20:37:24,868 DEBUG [RS:0;5be53b084ac7:44543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-11T20:37:24,868 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.LeaseManager(133): Closed leases 2024-12-11T20:37:24,868 INFO [RS:0;5be53b084ac7:44543 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T20:37:24,868 INFO [RS:0;5be53b084ac7:44543 {}] hbase.ChoreService(370): Chore service for: regionserver/5be53b084ac7:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-11T20:37:24,868 INFO [RS:0;5be53b084ac7:44543 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T20:37:24,868 INFO [regionserver/5be53b084ac7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T20:37:24,869 INFO [RS:0;5be53b084ac7:44543 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44543 2024-12-11T20:37:24,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-11T20:37:24,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5be53b084ac7,44543,1733949248752 2024-12-11T20:37:24,924 INFO [RS:0;5be53b084ac7:44543 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T20:37:24,924 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5be53b084ac7,44543,1733949248752] 2024-12-11T20:37:24,940 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5be53b084ac7,44543,1733949248752 already deleted, retry=false 2024-12-11T20:37:24,941 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5be53b084ac7,44543,1733949248752 expired; onlineServers=0 2024-12-11T20:37:24,941 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5be53b084ac7,39923,1733949248041' ***** 2024-12-11T20:37:24,941 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-11T20:37:24,941 INFO [M:0;5be53b084ac7:39923 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-11T20:37:24,941 INFO [M:0;5be53b084ac7:39923 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-11T20:37:24,941 DEBUG [M:0;5be53b084ac7:39923 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-11T20:37:24,941 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-11T20:37:24,941 DEBUG [M:0;5be53b084ac7:39923 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-11T20:37:24,941 DEBUG [master/5be53b084ac7:0:becomeActiveMaster-HFileCleaner.large.0-1733949251576 {}] cleaner.HFileCleaner(306): Exit Thread[master/5be53b084ac7:0:becomeActiveMaster-HFileCleaner.large.0-1733949251576,5,FailOnTimeoutGroup] 2024-12-11T20:37:24,941 DEBUG [master/5be53b084ac7:0:becomeActiveMaster-HFileCleaner.small.0-1733949251577 {}] cleaner.HFileCleaner(306): Exit Thread[master/5be53b084ac7:0:becomeActiveMaster-HFileCleaner.small.0-1733949251577,5,FailOnTimeoutGroup] 2024-12-11T20:37:24,942 INFO [M:0;5be53b084ac7:39923 {}] hbase.ChoreService(370): Chore service for: master/5be53b084ac7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-11T20:37:24,942 INFO [M:0;5be53b084ac7:39923 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-11T20:37:24,942 DEBUG [M:0;5be53b084ac7:39923 {}] master.HMaster(1795): Stopping service threads 2024-12-11T20:37:24,942 INFO [M:0;5be53b084ac7:39923 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-11T20:37:24,942 INFO [M:0;5be53b084ac7:39923 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-11T20:37:24,942 ERROR [M:0;5be53b084ac7:39923 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (169557626) connection to localhost/127.0.0.1:34529 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:34529,5,PEWorkerGroup] 2024-12-11T20:37:24,943 INFO [M:0;5be53b084ac7:39923 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-11T20:37:24,943 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-11T20:37:24,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-11T20:37:24,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-11T20:37:24,949 DEBUG [M:0;5be53b084ac7:39923 {}] zookeeper.ZKUtil(347): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-11T20:37:24,949 WARN [M:0;5be53b084ac7:39923 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-11T20:37:24,950 INFO [M:0;5be53b084ac7:39923 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/.lastflushedseqids 2024-12-11T20:37:24,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742460_1636 (size=72) 2024-12-11T20:37:25,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T20:37:25,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44543-0x10016a38d920001, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T20:37:25,033 INFO [RS:0;5be53b084ac7:44543 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T20:37:25,033 INFO [RS:0;5be53b084ac7:44543 {}] regionserver.HRegionServer(1031): Exiting; stopping=5be53b084ac7,44543,1733949248752; zookeeper connection closed. 2024-12-11T20:37:25,034 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@50428709 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@50428709 2024-12-11T20:37:25,034 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-11T20:37:25,364 INFO [M:0;5be53b084ac7:39923 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-11T20:37:25,365 INFO [M:0;5be53b084ac7:39923 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-11T20:37:25,365 DEBUG [M:0;5be53b084ac7:39923 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-11T20:37:25,365 INFO [M:0;5be53b084ac7:39923 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:37:25,365 DEBUG [M:0;5be53b084ac7:39923 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:37:25,365 DEBUG [M:0;5be53b084ac7:39923 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-11T20:37:25,365 DEBUG [M:0;5be53b084ac7:39923 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:37:25,365 INFO [M:0;5be53b084ac7:39923 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=792.20 KB heapSize=971.93 KB 2024-12-11T20:37:25,379 DEBUG [M:0;5be53b084ac7:39923 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/349b23289fb849729bb2bbc77f8000f7 is 82, key is hbase:meta,,1/info:regioninfo/1733949252986/Put/seqid=0 2024-12-11T20:37:25,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742461_1637 (size=5672) 2024-12-11T20:37:25,783 INFO [M:0;5be53b084ac7:39923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2228 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/349b23289fb849729bb2bbc77f8000f7 2024-12-11T20:37:25,817 DEBUG [M:0;5be53b084ac7:39923 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cca11060172040b0b367e8b5fef80860 is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1733949412292/Put/seqid=0 2024-12-11T20:37:25,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742462_1638 (size=45076) 2024-12-11T20:37:26,222 INFO [M:0;5be53b084ac7:39923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=791.65 KB at sequenceid=2228 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cca11060172040b0b367e8b5fef80860 2024-12-11T20:37:26,247 INFO [M:0;5be53b084ac7:39923 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cca11060172040b0b367e8b5fef80860 2024-12-11T20:37:26,265 DEBUG [M:0;5be53b084ac7:39923 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/56b9dd28485a43d5beb2357b1e8bb964 is 69, key is 5be53b084ac7,44543,1733949248752/rs:state/1733949251674/Put/seqid=0 2024-12-11T20:37:26,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073742463_1639 (size=5156) 2024-12-11T20:37:26,670 INFO [M:0;5be53b084ac7:39923 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2228 (bloomFilter=true), to=hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/56b9dd28485a43d5beb2357b1e8bb964 2024-12-11T20:37:26,680 DEBUG [M:0;5be53b084ac7:39923 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/349b23289fb849729bb2bbc77f8000f7 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/349b23289fb849729bb2bbc77f8000f7 2024-12-11T20:37:26,684 INFO [M:0;5be53b084ac7:39923 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/349b23289fb849729bb2bbc77f8000f7, entries=8, sequenceid=2228, filesize=5.5 K 2024-12-11T20:37:26,685 DEBUG [M:0;5be53b084ac7:39923 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cca11060172040b0b367e8b5fef80860 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cca11060172040b0b367e8b5fef80860 2024-12-11T20:37:26,688 INFO [M:0;5be53b084ac7:39923 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cca11060172040b0b367e8b5fef80860 2024-12-11T20:37:26,688 INFO [M:0;5be53b084ac7:39923 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cca11060172040b0b367e8b5fef80860, entries=168, sequenceid=2228, filesize=44.0 K 2024-12-11T20:37:26,689 DEBUG [M:0;5be53b084ac7:39923 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/56b9dd28485a43d5beb2357b1e8bb964 as hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/56b9dd28485a43d5beb2357b1e8bb964 2024-12-11T20:37:26,692 INFO [M:0;5be53b084ac7:39923 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34529/user/jenkins/test-data/e9bb5b41-de56-4ce5-3cde-b4db155e9043/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/56b9dd28485a43d5beb2357b1e8bb964, entries=1, sequenceid=2228, filesize=5.0 K 2024-12-11T20:37:26,692 INFO [M:0;5be53b084ac7:39923 {}] regionserver.HRegion(3140): Finished flush of dataSize ~792.20 KB/811216, heapSize ~971.63 KB/994952, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1327ms, sequenceid=2228, compaction requested=false 2024-12-11T20:37:26,694 INFO [M:0;5be53b084ac7:39923 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-11T20:37:26,694 DEBUG [M:0;5be53b084ac7:39923 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733949445365Disabling compacts and flushes for region at 1733949445365Disabling writes for close at 1733949445365Obtaining lock to block concurrent updates at 1733949445365Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733949445365Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=811216, getHeapSize=995192, getOffHeapSize=0, getCellsCount=2421 at 1733949445365Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733949445366 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733949445366Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733949445378 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733949445378Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733949445792 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733949445816 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733949445816Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733949446247 (+431 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733949446265 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733949446265Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2882ee53: reopening flushed file at 1733949446678 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68b71199: reopening flushed file at 1733949446684 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e996bfa: reopening flushed file at 1733949446688 (+4 ms)Finished flush of dataSize ~792.20 KB/811216, heapSize ~971.63 KB/994952, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1327ms, sequenceid=2228, compaction requested=false at 1733949446692 (+4 ms)Writing region close event to WAL at 1733949446694 (+2 ms)Closed at 1733949446694 2024-12-11T20:37:26,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35965 is added to blk_1073741830_1006 (size=957227) 2024-12-11T20:37:26,697 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-11T20:37:26,697 INFO [M:0;5be53b084ac7:39923 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-11T20:37:26,697 INFO [M:0;5be53b084ac7:39923 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39923 2024-12-11T20:37:26,697 INFO [M:0;5be53b084ac7:39923 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-11T20:37:26,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T20:37:26,824 INFO [M:0;5be53b084ac7:39923 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-11T20:37:26,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39923-0x10016a38d920000, quorum=127.0.0.1:59301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-11T20:37:26,856 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3874740c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-11T20:37:26,861 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@589acbe1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T20:37:26,861 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T20:37:26,862 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@55a78c31{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T20:37:26,862 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@15953f85{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/hadoop.log.dir/,STOPPED} 2024-12-11T20:37:26,865 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-11T20:37:26,865 WARN [BP-2058600437-172.17.0.2-1733949244852 heartbeating to localhost/127.0.0.1:34529 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-11T20:37:26,865 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-11T20:37:26,865 WARN [BP-2058600437-172.17.0.2-1733949244852 heartbeating to localhost/127.0.0.1:34529 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2058600437-172.17.0.2-1733949244852 (Datanode Uuid a72c7df4-e10e-48ba-9eff-506085311529) service to localhost/127.0.0.1:34529 2024-12-11T20:37:26,867 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/cluster_cff584a6-4744-f919-71a4-2a9a95c09adc/data/data1/current/BP-2058600437-172.17.0.2-1733949244852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T20:37:26,867 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/cluster_cff584a6-4744-f919-71a4-2a9a95c09adc/data/data2/current/BP-2058600437-172.17.0.2-1733949244852 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-11T20:37:26,867 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-11T20:37:26,873 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@55a6f9c7{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-11T20:37:26,874 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@40742525{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-11T20:37:26,874 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-11T20:37:26,874 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6efc0ba4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-11T20:37:26,874 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4516de21{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/c8ebb110-63da-cfaf-56d3-59caf15644d9/hadoop.log.dir/,STOPPED} 2024-12-11T20:37:26,888 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-11T20:37:27,016 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down